From 0632a97f55af67c7e5265b0d3283155c7c087e89 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Thu, 26 Nov 2020 10:42:10 -0800 Subject: [PATCH] Simplified sidecar table in export_db --- osxphotos/export_db.py | 91 ++++-------------------- osxphotos/photoinfo/_photoinfo_export.py | 8 +-- tests/test_export_db.py | 45 ++++-------- 3 files changed, 29 insertions(+), 115 deletions(-) diff --git a/osxphotos/export_db.py b/osxphotos/export_db.py index 45f44ca2..e7714baf 100644 --- a/osxphotos/export_db.py +++ b/osxphotos/export_db.py @@ -14,7 +14,7 @@ from sqlite3 import Error from ._version import __version__ -OSXPHOTOS_EXPORTDB_VERSION = "3.1" +OSXPHOTOS_EXPORTDB_VERSION = "3.2" class ExportDB_ABC(ABC): @@ -77,19 +77,11 @@ class ExportDB_ABC(ABC): pass @abstractmethod - def set_exiftool_json_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): + def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): pass @abstractmethod - def get_exiftool_json_sidecar_for_file(self, filename): - pass - - @abstractmethod - def set_xmp_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): - pass - - @abstractmethod - def get_xmp_sidecar_for_file(self, filename): + def get_sidecar_for_file(self, filename): pass @abstractmethod @@ -157,16 +149,10 @@ class ExportDBNoOp(ExportDB_ABC): def set_exifdata_for_file(self, uuid, exifdata): pass - def set_exiftool_json_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): + def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): pass - def get_exiftool_json_sidecar_for_file(self, filename): - return None, (None, None, None) - - def set_xmp_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): - pass - - def get_xmp_sidecar_for_file(self, filename): + def get_sidecar_for_file(self, filename): return None, (None, None, None) def set_data( @@ -407,14 +393,14 @@ class ExportDB(ExportDB_ABC): except Error as e: logging.warning(e) - def get_exiftool_json_sidecar_for_file(self, filename): - """ returns the exiftool JSON sidecar data and signature for a file """ + def get_sidecar_for_file(self, filename): + """ returns the sidecar data and signature for a file """ filename = str(pathlib.Path(filename).relative_to(self._path)).lower() conn = self._conn try: c = conn.cursor() c.execute( - "SELECT sidecar_data, mode, size, mtime FROM exiftool_json_sidecar WHERE filepath_normalized = ?", + "SELECT sidecar_data, mode, size, mtime FROM sidecar WHERE filepath_normalized = ?", (filename,), ) results = c.fetchone() @@ -435,56 +421,14 @@ class ExportDB(ExportDB_ABC): return sidecar_data, sidecar_sig - def set_exiftool_json_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): - """ sets the exiftool JSON sidecar data and signature for a file """ + def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): + """ sets the sidecar data and signature for a file """ filename = str(pathlib.Path(filename).relative_to(self._path)).lower() conn = self._conn try: c = conn.cursor() c.execute( - "INSERT OR REPLACE INTO exiftool_json_sidecar(filepath_normalized, sidecar_data, mode, size, mtime) VALUES (?, ?, ?, ?, ?);", - (filename, sidecar_data, *sidecar_sig), - ) - conn.commit() - except Error as e: - logging.warning(e) - - def get_xmp_sidecar_for_file(self, filename): - """ returns the XMP sidecar data and signature for a file """ - filename = str(pathlib.Path(filename).relative_to(self._path)).lower() - conn = self._conn - try: - c = conn.cursor() - c.execute( - "SELECT sidecar_data, mode, size, mtime FROM xmp_sidecar WHERE filepath_normalized = ?", - (filename,), - ) - results = c.fetchone() - if results: - sidecar_data = results[0] - sidecar_sig = ( - results[1], - results[2], - int(results[3]) if results[3] is not None else None, - ) - else: - sidecar_data = None - sidecar_sig = (None, None, None) - except Error as e: - logging.warning(e) - sidecar_data = None - sidecar_sig = (None, None, None) - - return sidecar_data, sidecar_sig - - def set_xmp_sidecar_for_file(self, filename, sidecar_data, sidecar_sig): - """ sets the XMP sidecar data for a file """ - filename = str(pathlib.Path(filename).relative_to(self._path)).lower() - conn = self._conn - try: - c = conn.cursor() - c.execute( - "INSERT OR REPLACE INTO xmp_sidecar(filepath_normalized, sidecar_data, mode, size, mtime) VALUES (?, ?, ?, ?, ?);", + "INSERT OR REPLACE INTO sidecar(filepath_normalized, sidecar_data, mode, size, mtime) VALUES (?, ?, ?, ?, ?);", (filename, sidecar_data, *sidecar_sig), ) conn.commit() @@ -679,7 +623,7 @@ class ExportDB(ExportDB_ABC): size INTEGER, mtime REAL ); """, - "sql_xmp_table": """ CREATE TABLE IF NOT EXISTS xmp_sidecar ( + "sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar ( id INTEGER PRIMARY KEY, filepath_normalized TEXT NOT NULL, sidecar_data TEXT, @@ -687,21 +631,12 @@ class ExportDB(ExportDB_ABC): size INTEGER, mtime REAL ); """, - "sql_exiftool_json_table": """ CREATE TABLE IF NOT EXISTS exiftool_json_sidecar ( - id INTEGER PRIMARY KEY, - filepath_normalized TEXT NOT NULL, - sidecar_data JSON, - mode INTEGER, - size INTEGER, - mtime REAL - ); """, "sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """, "sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """, "sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """, "sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""", "sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""", - "sql_xmp_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_xmp_filename on xmp_sidecar (filepath_normalized);""", - "sql_exiftool_json_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exiftool_json_filename on exiftool_json_sidecar (filepath_normalized);""", + "sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""", } try: c = conn.cursor() diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index c4434890..8ad9f4ac 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -760,7 +760,7 @@ def export2( ignore_date_modified=ignore_date_modified, ) sidecar_digest = hexdigest(sidecar_str) - old_sidecar_digest, sidecar_sig = export_db.get_exiftool_json_sidecar_for_file( + old_sidecar_digest, sidecar_sig = export_db.get_sidecar_for_file( sidecar_filename ) write_sidecar = ( @@ -776,7 +776,7 @@ def export2( verbose(f"Writing exiftool JSON sidecar {sidecar_filename}") if not dry_run: self._write_sidecar(sidecar_filename, sidecar_str) - export_db.set_exiftool_json_sidecar_for_file( + export_db.set_sidecar_for_file( sidecar_filename, sidecar_digest, fileutil.file_sig(sidecar_filename), @@ -796,7 +796,7 @@ def export2( extension=dest.suffix[1:] if dest.suffix else None, ) sidecar_digest = hexdigest(sidecar_str) - old_sidecar_digest, sidecar_sig = export_db.get_xmp_sidecar_for_file( + old_sidecar_digest, sidecar_sig = export_db.get_sidecar_for_file( sidecar_filename ) write_sidecar = ( @@ -812,7 +812,7 @@ def export2( verbose(f"Writing XMP sidecar {sidecar_filename}") if not dry_run: self._write_sidecar(sidecar_filename, sidecar_str) - export_db.set_xmp_sidecar_for_file( + export_db.set_sidecar_for_file( sidecar_filename, sidecar_digest, fileutil.file_sig(sidecar_filename), diff --git a/tests/test_export_db.py b/tests/test_export_db.py index fc2ee7b5..e65b42cc 100644 --- a/tests/test_export_db.py +++ b/tests/test_export_db.py @@ -4,8 +4,7 @@ import pytest EXIF_DATA = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "EXIF:ImageDescription": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Title": "Elder Park", "EXIF:GPSLatitude": "34 deg 55' 8.01\" S", "EXIF:GPSLongitude": "138 deg 35' 48.70\" E", "Composite:GPSPosition": "34 deg 55' 8.01\" S, 138 deg 35' 48.70\" E", "EXIF:GPSLatitudeRef": "South", "EXIF:GPSLongitudeRef": "East", "EXIF:DateTimeOriginal": "2017:06:20 17:18:56", "EXIF:OffsetTimeOriginal": "+09:30", "EXIF:ModifyDate": "2020:05:18 14:42:04"}]""" INFO_DATA = """{"uuid": "3DD2C897-F19E-4CA6-8C22-B027D5A71907", "filename": "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "original_filename": "IMG_4547.jpg", "date": "2017-06-20T17:18:56.518000+09:30", "description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "title": "Elder Park", "keywords": [], "labels": ["Statue", "Art"], "albums": ["AlbumInFolder"], "folders": {"AlbumInFolder": ["Folder1", "SubFolder2"]}, "persons": [], "path": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/originals/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "ismissing": false, "hasadjustments": true, "external_edit": false, "favorite": false, "hidden": false, "latitude": -34.91889167000001, "longitude": 138.59686167, "path_edited": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/resources/renders/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907_1_201_a.jpeg", "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null, "date_modified": "2020-05-18T14:42:04.608664+09:30", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Elder Park, Adelaide, South Australia, Australia, River Torrens", "names": {"field0": [], "country": ["Australia"], "state_province": ["South Australia"], "sub_administrative_area": ["Adelaide"], "city": ["Adelaide", "Adelaide"], "field5": [], "additional_city_info": ["Adelaide CBD", "Tarndanya"], "ocean": [], "area_of_interest": ["Elder Park", ""], "inland_water": ["River Torrens", "River Torrens"], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": ["River Torrens", "River Torrens"]}, "country_code": "AU", "ishome": false, "address_str": "River Torrens, Adelaide SA, Australia", "address": {"street": null, "sub_locality": "Tarndanya", "city": "Adelaide", "sub_administrative_area": "Adelaide", "state_province": "SA", "postal_code": null, "country": "Australia", "iso_country_code": "AU"}}, "exif": {"flash_fired": false, "iso": 320, "metering_mode": 3, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.058823529411764705, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}""" -EXIFTOOL_JSON_DATA = """FOO_BAR""" -XMP_DATA = """FOO_BAR""" +SIDECAR_DATA = """FOO_BAR""" EXIF_DATA2 = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "XMP:Title": "St. James's Park", "XMP:TagsList": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "IPTC:Keywords": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "XMP:Subject": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "EXIF:GPSLatitude": "51 deg 30' 12.86\" N", "EXIF:GPSLongitude": "0 deg 7' 54.50\" W", "Composite:GPSPosition": "51 deg 30' 12.86\" N, 0 deg 7' 54.50\" W", "EXIF:GPSLatitudeRef": "North", "EXIF:GPSLongitudeRef": "West", "EXIF:DateTimeOriginal": "2018:10:13 09:18:12", "EXIF:OffsetTimeOriginal": "-04:00", "EXIF:ModifyDate": "2019:12:08 14:06:44"}]""" INFO_DATA2 = """{"uuid": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529", "filename": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "original_filename": "IMG_8440.JPG", "date": "2019-06-11T11:42:06.711805-07:00", "description": null, "title": null, "keywords": [], "labels": ["Sky", "Cloudy", "Fence", "Land", "Outdoor", "Park", "Amusement Park", "Roller Coaster"], "albums": [], "folders": {}, "persons": [], "path": "/Volumes/MacBook Catalina - Data/Users/rhet/Pictures/Photos Library.photoslibrary/originals/F/F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 33.81558666666667, "longitude": -117.99298, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": true, "incloud": true, "date_modified": "2019-10-14T00:51:47.141950-07:00", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Adventure City, Stanton, California, United States", "names": {"field0": [], "country": ["United States"], "state_province": ["California"], "sub_administrative_area": ["Orange"], "city": ["Stanton", "Anaheim", "Anaheim"], "field5": [], "additional_city_info": ["West Anaheim"], "ocean": [], "area_of_interest": ["Adventure City", "Adventure City"], "inland_water": [], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": []}, "country_code": "US", "ishome": false, "address_str": "Adventure City, 1240 S Beach Blvd, Anaheim, CA 92804, United States", "address": {"street": "1240 S Beach Blvd", "sub_locality": "West Anaheim", "city": "Stanton", "sub_administrative_area": "Orange", "state_province": "CA", "postal_code": "92804", "country": "United States", "iso_country_code": "US"}}, "exif": {"flash_fired": false, "iso": 25, "metering_mode": 5, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.0004940711462450593, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}""" @@ -44,13 +43,8 @@ def test_export_db(): assert db.get_stat_edited_for_file(filepath) == (10, 11, 12) db.set_stat_converted_for_file(filepath, (7, 8, 9)) assert db.get_stat_converted_for_file(filepath) == (7, 8, 9) - db.set_exiftool_json_sidecar_for_file(filepath, EXIFTOOL_JSON_DATA, (10, 11, 12)) - assert db.get_exiftool_json_sidecar_for_file(filepath) == ( - EXIFTOOL_JSON_DATA, - (10, 11, 12), - ) - db.set_xmp_sidecar_for_file(filepath, XMP_DATA, (13, 14, 15)) - assert db.get_xmp_sidecar_for_file(filepath) == (XMP_DATA, (13, 14, 15)) + db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15)) + assert db.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15)) # test set_data which sets all at the same time filepath2 = os.path.join(tempdir.name, "test2.jpg") @@ -119,10 +113,8 @@ def test_export_db_no_op(): assert db.get_stat_converted_for_file(filepath) is None db.set_stat_edited_for_file(filepath, (10, 11, 12)) assert db.get_stat_edited_for_file(filepath) is None - db.set_exiftool_json_sidecar_for_file(filepath, EXIFTOOL_JSON_DATA, (10, 11, 12)) - assert db.get_exiftool_json_sidecar_for_file(filepath) == (None, (None, None, None)) - db.set_xmp_sidecar_for_file(filepath, XMP_DATA, (13, 14, 15)) - assert db.get_xmp_sidecar_for_file(filepath) == (None, (None, None, None)) + db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15)) + assert db.get_sidecar_for_file(filepath) == (None, (None, None, None)) # test set_data which sets all at the same time filepath2 = os.path.join(tempdir.name, "test2.jpg") @@ -174,8 +166,7 @@ def test_export_db_in_memory(): db.set_stat_exif_for_file(filepath, (4, 5, 6)) db.set_stat_converted_for_file(filepath, (7, 8, 9)) db.set_stat_edited_for_file(filepath, (10, 11, 12)) - db.set_exiftool_json_sidecar_for_file(filepath, EXIFTOOL_JSON_DATA, (10, 11, 12)) - db.set_xmp_sidecar_for_file(filepath, XMP_DATA, (13, 14, 15)) + db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15)) db.close() @@ -192,11 +183,7 @@ def test_export_db_in_memory(): assert dbram.get_stat_exif_for_file(filepath) == (4, 5, 6) assert dbram.get_stat_converted_for_file(filepath) == (7, 8, 9) assert dbram.get_stat_edited_for_file(filepath) == (10, 11, 12) - assert dbram.get_exiftool_json_sidecar_for_file(filepath) == ( - EXIFTOOL_JSON_DATA, - (10, 11, 12), - ) - assert dbram.get_xmp_sidecar_for_file(filepath) == (XMP_DATA, (13, 14, 15)) + assert dbram.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15)) # change a value dbram.set_uuid_for_file(filepath, "FUBAR") @@ -206,8 +193,7 @@ def test_export_db_in_memory(): dbram.set_stat_exif_for_file(filepath, (10, 11, 12)) dbram.set_stat_converted_for_file(filepath, (1, 2, 3)) dbram.set_stat_edited_for_file(filepath, (4, 5, 6)) - dbram.set_exiftool_json_sidecar_for_file(filepath, "FUBAR", (17, 18, 19)) - dbram.set_xmp_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) + dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR" assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2 @@ -216,8 +202,7 @@ def test_export_db_in_memory(): assert dbram.get_stat_exif_for_file(filepath) == (10, 11, 12) assert dbram.get_stat_converted_for_file(filepath) == (1, 2, 3) assert dbram.get_stat_edited_for_file(filepath) == (4, 5, 6) - assert dbram.get_exiftool_json_sidecar_for_file(filepath) == ("FUBAR", (17, 18, 19)) - assert dbram.get_xmp_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) + assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) dbram.close() @@ -230,11 +215,7 @@ def test_export_db_in_memory(): assert db.get_stat_exif_for_file(filepath) == (4, 5, 6) assert db.get_stat_converted_for_file(filepath) == (7, 8, 9) assert db.get_stat_edited_for_file(filepath) == (10, 11, 12) - assert db.get_exiftool_json_sidecar_for_file(filepath) == ( - EXIFTOOL_JSON_DATA, - (10, 11, 12), - ) - assert db.get_xmp_sidecar_for_file(filepath) == (XMP_DATA, (13, 14, 15)) + assert db.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15)) assert db.get_info_for_uuid("FUBAR") is None @@ -262,8 +243,7 @@ def test_export_db_in_memory_nofile(): dbram.set_stat_exif_for_file(filepath, (10, 11, 12)) dbram.set_stat_converted_for_file(filepath, (1, 2, 3)) dbram.set_stat_edited_for_file(filepath, (4, 5, 6)) - dbram.set_exiftool_json_sidecar_for_file(filepath, "FUBAR", (17, 18, 19)) - dbram.set_xmp_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) + dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR" assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2 @@ -272,7 +252,6 @@ def test_export_db_in_memory_nofile(): assert dbram.get_stat_exif_for_file(filepath) == (10, 11, 12) assert dbram.get_stat_converted_for_file(filepath) == (1, 2, 3) assert dbram.get_stat_edited_for_file(filepath) == (4, 5, 6) - assert dbram.get_exiftool_json_sidecar_for_file(filepath) == ("FUBAR", (17, 18, 19)) - assert dbram.get_xmp_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) + assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) dbram.close()