Mercurial > lbo > hg > photosync
changeset 0:7d1e69d4e2ac
Initial commit
| author | Lewin Bormann <lbo@spheniscida.de> |
|---|---|
| date | Fri, 14 Jun 2019 13:59:20 +0200 |
| parents | |
| children | a119256589bc |
| files | Pipfile Pipfile.lock photosync.py |
| diffstat | 3 files changed, 335 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Pipfile Fri Jun 14 13:59:20 2019 +0200 @@ -0,0 +1,15 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +google-api-python-client = "*" +google-auth-httplib2 = "*" +google-auth-oauthlib = "*" +python-dateutil = "*" + +[requires] +python_version = "3.7"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Pipfile.lock Fri Jun 14 13:59:20 2019 +0200 @@ -0,0 +1,159 @@ +{ + "_meta": { + "hash": { + "sha256": "92e1fcff9dc3bf8ed8a7de79e9fc8f419f16b9644ec2bf80975549f81c296044" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "cachetools": { + "hashes": [ + "sha256:428266a1c0d36dc5aca63a2d7c5942e88c2c898d72139fca0e97fdd2380517ae", + "sha256:8ea2d3ce97850f31e4a08b0e2b5e6c34997d7216a9d2c98e0f3978630d4da69a" + ], + "version": "==3.1.1" + }, + "certifi": { + "hashes": [ + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + ], + "version": "==2019.3.9" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "google-api-python-client": { + "hashes": [ + "sha256:048da0d68564380ee23b449e5a67d4666af1b3b536d2fb0a02cee1ad540fa5ec", + "sha256:5def5a485b1cbc998b8f869456c7bde0c0e6d3d0a5ea1f300b5ef57cb4b1ce8f" + ], + "index": "pypi", + "version": "==1.7.9" + }, + "google-auth": { + "hashes": [ + "sha256:0f7c6a64927d34c1a474da92cfc59e552a5d3b940d3266606c6a28b72888b9e4", + "sha256:20705f6803fd2c4d1cc2dcb0df09d4dfcb9a7d51fd59e94a3a28231fd93119ed" + ], + "version": "==1.6.3" + }, + "google-auth-httplib2": { + "hashes": [ + "sha256:098fade613c25b4527b2c08fa42d11f3c2037dda8995d86de0745228e965d445", + "sha256:f1c437842155680cf9918df9bc51c1182fda41feef88c34004bd1978c8157e08" + ], + "index": "pypi", + "version": "==0.0.3" + }, + "google-auth-oauthlib": { + "hashes": [ + "sha256:6a8b0072048940d1f41c23c03576867e577e826fec140a1c7e148ec486e083ba", + "sha256:904d72541fc92adb2026767dd364ab3a51f6e893d64b13cdc6acbd580c841468" + ], + "index": "pypi", + "version": "==0.4.0" + }, + "httplib2": { + "hashes": [ + "sha256:158fbd0ffbba536829d664bf3f32c4f45df41f8f791663665162dfaf21ffd075", + "sha256:d1146939d270f1f1eb8cbf8f5aa72ff37d897faccca448582bb1e180aeb4c6b2" + ], + "version": "==0.13.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "oauthlib": { + "hashes": [ + "sha256:0ce32c5d989a1827e3f1148f98b9085ed2370fc939bf524c9c851d8714797298", + "sha256:3e1e14f6cde7e5475128d30e97edc3bfb4dc857cb884d8714ec161fdbb3b358e" + ], + "version": "==3.0.1" + }, + "pyasn1": { + "hashes": [ + "sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7", + "sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e" + ], + "version": "==0.4.5" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:ef721f68f7951fab9b0404d42590f479e30d9005daccb1699b0a51bb4177db96", + "sha256:f309b6c94724aeaf7ca583feb1cc70430e10d7551de5e36edfc1ae6909bcfb3c" + ], + "version": "==0.2.5" + }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "index": "pypi", + "version": "==2.8.0" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "requests-oauthlib": { + "hashes": [ + "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57", + "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140" + ], + "version": "==1.2.0" + }, + "rsa": { + "hashes": [ + "sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66", + "sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487" + ], + "version": "==4.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "uritemplate": { + "hashes": [ + "sha256:01c69f4fe8ed503b2951bef85d996a9d22434d2431584b5b107b2981ff416fbd", + "sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd", + "sha256:c02643cebe23fc8adb5e6becffe201185bf06c40bda5c0b4028a93f1527d011d" + ], + "version": "==3.0.0" + }, + "urllib3": { + "hashes": [ + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + ], + "version": "==1.25.3" + } + }, + "develop": {} +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/photosync.py Fri Jun 14 13:59:20 2019 +0200 @@ -0,0 +1,161 @@ + +import datetime +import dateutil.parser +import json +import os.path +import pickle +import sqlite3 +import urllib3 +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request + +class TokenSource: + SCOPES = ['https://www.googleapis.com/auth/photoslibrary'] + CRED_ID = 'installed.main' + + def __init__(self, db=None, tokensfile=None, clientsecret='clientsecret.json'): + self._db = db + self._tokensfile = tokensfile + self._clientsecret = clientsecret + + def creds(self): + if self._tokensfile and os.path.exists(self._tokensfile): + with open(self._tokensfile, 'rb') as f: + creds = pickle.load(f) + return creds + elif self._db: + creds = self._db.get_credentials(self.CRED_ID) + if creds: + creds = pickle.loads(creds) + return creds + flow = InstalledAppFlow.from_client_secrets_file(self._clientsecret, self.SCOPES) + creds = flow.run_local_server() + if creds and self._tokensfile: + with open(self._tokensfile, 'wb') as f: + pickle.dump(creds, f) + if creds and self._db: + self._db.store_credentials(self.CRED_ID, pickle.dumps(creds)) + return creds + +class PhotosService: + + def __init__(self, tokens=None): + self._token_source = tokens + self._service = build('photoslibrary', 'v1', credentials=tokens.creds()) + self._http = urllib3.PoolManager() + + def list_library(self, start=None, to=None): + """Yields items from the library. + + Arguments: + start: datetime.date + end: datetime.date + + Returns: + [mediaItem] + """ + filters = {} + if start or to: + rng_filter = {'ranges': {}} + if start: + rng_filter['ranges']['startDate'] = {'year': start.year, 'month': start.month, 'day': start.day} + else: + rng_filter['ranges']['startDate'] = {'year': 1999, 'month': 1, 'day': 1} + if not to: + to = datetime.datetime.now().date() + rng_filter['ranges']['endDate'] = {'year': to.year, 'month': to.month, 'day': to.day} + filters['dateFilter'] = rng_filter + pagetoken = None + while True: + resp = self._service.mediaItems().search(body={'pageSize': 25, 'filters': filters, 'pageToken': pagetoken}).execute() + items = resp['mediaItems'] + pagetoken = resp.get('nextPageToken', None) + for i in items: + yield i + if pagetoken is None: + return + + def download_photo(self, id, path): + """Download a photo and store it under its file name in the directory `path`. + """ + photo = self._service.mediaItems().get(mediaItemId=id).execute() + rawurl = photo['baseUrl'] + p = os.path.join(path, photo['filename']) + with open(p, 'wb') as f: + f.write(self._http.request('GET', rawurl).data) + +class DB: + + def __init__(self, path): + self._db = sqlite3.connect(path) + self.initdb() + self._dtparse = dateutil.parser.isoparser() + + def initdb(self): + cur = self._db.cursor() + cur.execute('CREATE TABLE IF NOT EXISTS photos (id TEXT PRIMARY KEY, creationTime TEXT, path TEXT, filename TEXT, offline INTEGER)') + cur.execute('CREATE TABLE IF NOT EXISTS transactions (id TEXT, type TEXT, time INTEGER, path TEXT, filename TEXT)') + cur.execute('CREATE TABLE IF NOT EXISTS oauth (id TEXT PRIMARY KEY, credentials BLOB)') + self._db.commit() + + def store_credentials(self, id, creds): + with self._db as conn: + cur = conn.cursor() + cur.execute('SELECT id FROM oauth WHERE id = ?', (id,)) + if not cur.fetchone(): + cur.execute('INSERT INTO oauth (id, credentials) VALUES (?, ?)', (id, creds)) + return + cur.close() + cur = conn.cursor() + cur.execute('UPDATE oauth SET credentials = ? WHERE id = ?', (creds, id)) + + def get_credentials(self, id): + with self._db as conn: + cur = conn.cursor() + cur.execute('SELECT credentials FROM oauth WHERE id = ?', (id,)) + row = cur.fetchone() + if row: + return row[0] + return None + + def add_online_photo(self, media_item, path): + with self._db as conn: + cur = conn.cursor() + cur.execute('SELECT id FROM photos WHERE id = "{}"'.format(media_item['id'])) + if cur.fetchone(): + print('WARN: Photo already in store.') + cur.close() + return + cur.close() + + creation_time = int(self._dtparse.isoparse(media_item['mediaMetadata']['creationTime']).timestamp()) + conn.cursor().execute( + 'INSERT INTO photos (id, creationTime, path, filename, offline) VALUES (?, ?, ?, ?, 0)', + media_item['id'], creation_time, path, media_item['filename']) + conn.commit() + + def mark_photo_downloaded(self, id): + with self._db as conn: + conn.cursor().execute( + 'UPDATE photos SET offline = 1 WHERE id = {}'.format(id)) + + def most_recent_creation_date(self): + with self._db as conn: + cursor = conn.cursor() + cursor.execute('SELECT creationTime FROM photos ORDER BY creationTime DESC LIMIT 1') + row = cursor.fetchone() + cursor.close() + if row: + return datetime.datetime.fromtimestamp(row[0]) + return datetime.datetime.fromtimestamp(0) + +def main(): + db = DB('sq.lite') + s = PhotosService(tokens=TokenSource(db=db)) + items = s.list_library(to=datetime.date(2019, 7, 7)) + for i in items: + db.add_online_photo(i, 'local') + +if __name__ == '__main__': + main()