changeset 15:f941f1b0fa09

Implement resync option to detect deleted photos.
author Lewin Bormann <lbo@spheniscida.de>
date Sat, 15 Jun 2019 11:03:46 +0200
parents b83a9e2622b3
children 53e94b301d31
files photosync.py
diffstat 1 files changed, 45 insertions(+), 8 deletions(-) [+]
line wrap: on
line diff
--- a/photosync.py	Sat Jun 15 10:49:09 2019 +0200
+++ b/photosync.py	Sat Jun 15 11:03:46 2019 +0200
@@ -146,6 +146,7 @@
             ok.append(item['id'])
         return ok
 
+
 class DB:
 
     def __init__(self, path):
@@ -198,21 +199,21 @@
         self.record_transaction(media_item['id'], 'ADD')
         return True
 
-    def get_not_downloaded_items(self):
+    def get_items_by_downloaded(self, downloaded=False):
         """Generate items (as [id, path, filename, is_video]) that are not yet present locally."""
         with self._db as conn:
             cur = conn.cursor()
-            cur.execute('SELECT id, path, filename, video FROM items WHERE offline = 0 ORDER BY creationTime ASC')
+            cur.execute('SELECT id, path, filename, video FROM items WHERE offline = ? ORDER BY creationTime ASC', (1 if downloaded else 0,))
             while True:
                 row = cur.fetchone()
                 if not row:
                     break
                 yield row
 
-    def mark_items_downloaded(self, ids):
+    def mark_items_downloaded(self, ids, downloaded=True):
         with self._db as conn:
             for id in ids:
-                conn.cursor().execute('UPDATE items SET offline = 1 WHERE id = ?', (id,))
+                conn.cursor().execute('UPDATE items SET offline = ? WHERE id = ?', (1 if downloaded else 0, id))
                 self.record_transaction(id, 'DOWNLOAD')
 
     def existing_items_range(self):
@@ -288,21 +289,27 @@
 
     def download_items(self):
         """Scans database for items not yet downloaded and downloads them."""
+        retry = []
         chunk = []
         chunksize = 16
-        for item in self._db.get_not_downloaded_items():
+        for item in self._db.get_items_by_downloaded(False):
             (id, path, filename, is_video) = item
             path = os.path.join(self._root, path)
             chunk.append((id, path, is_video))
 
             if len(chunk) > chunksize:
                 ok = self._svc.download_items(chunk)
+                wantids = list(map(lambda i: i[0], chunk))
                 self._db.mark_items_downloaded(ok)
                 chunk = []
+                retry.extend(set(wantids) ^ set(ok))
 
-        if len(chunk) > 0:
+        if len(chunk) + len(retry) > 0:
+            chunk.extend(retry)
             ok = self._svc.download_items(chunk)
             self._db.mark_items_downloaded(ok)
+            if len(ok) < len(chunk):
+                log('WARN', 'Could not download {} items. Please try again later (photosync will automatically retry these)', len(chunk) - len(ok))
 
     def drive(self, date_range=(None, None), window_heuristic=True):
         """First, download all metadata since most recently fetched item.
@@ -315,8 +322,31 @@
         if self.fetch_metadata(date_range, window_heuristic):
             self.download_items()
 
+    def find_vanished_items(self, dir):
+        """Checks if all photos that are supposed to be downloaded are still present.
+
+        Marks them for download otherwise, meaning that they will be downloaded later.
+        """
+        found = 0
+        for (id, path, filename, video) in self._db.get_items_by_downloaded(downloaded=True):
+            path = os.path.join(dir, path, filename)
+            try:
+                info = os.stat(path)
+            except FileNotFoundError:
+                log('INFO', 'Found vanished item at {}; marking for download', path)
+                found += 1
+                self._db.mark_items_downloaded([id], downloaded=False)
+        if found > 0:
+            log('WARN', 'Found {} vanished items. Reattempting download now...', found)
+            return True
+        return False
+
+
     def path_from_date(item):
-        """By default, map items to year/month/day directory."""
+        """By default, map items to year/month/day directory.
+
+        Important: Omits the --dir relative directory (self._root).
+        """
         dt = dateutil.parser.isoparser().isoparse(item['mediaMetadata']['creationTime']).date()
         return '{y}/{m:02d}/{d:02d}/'.format(y=dt.year, m=dt.month, d=dt.day)
 
@@ -332,8 +362,9 @@
         Options:
             -h --help                   Show this screen
             -d --dir=<dir>              Root directory; where to download photos and store the database.
+            --all                       Synchronize *all* photos instead of just before the oldest/after the newest photo. Needed if you have uploaded photos somewhere in the middle.
             --creds=clientsecret.json   Path to the client credentials JSON file. Defaults to
-            --all                       Synchronize *all* photos instead of just before the oldest/after the newest photo. Needed if you have uploaded photos somewhere in the middle.
+            --resync                    Check local filesystem for files that should be downloaded but are not there (anymore).
         '''
         super(arguments.BaseArguments, self).__init__(doc=doc)
         self.dir = self.dir or '.'
@@ -344,6 +375,12 @@
         db = DB(os.path.join(self.dir, 'sync.db'))
         s = PhotosService(tokens=TokenSource(db=db, clientsecret=self.creds))
         d = Driver(db, s, root=self.dir)
+
+        if self.resync:
+            if d.find_vanished_items(self.dir):
+                d.download_items()
+                log('WARN', 'Finished downloading missing items.')
+            return
         if self.all:
             d.drive(window_heuristic=False)
         else: