Implemented --ignore-signature, issue #286

README.md

@@ -236,6 +236,15 @@ Options:
                                   Deleted' folder.
   --update                        Only export new or updated files. See notes
                                   below on export and --update.
+  --ignore-signature              When used with --update, ignores file
+                                  signature when updating files. This is
+                                  useful if you have processed or edited
+                                  exported photos changing the file signature
+                                  (size & modification date). In this case,
+                                  --update would normally re-export the
+                                  processed files but with --ignore-signature,
+                                  files which exist in the export directory
+                                  will not be re-exported.
   --dry-run                       Dry run (test) the export but don't actually
                                   export any files; most useful with
                                   --verbose.
@@ -439,7 +448,10 @@ the export folder.  If a file is changed in the export folder (for example,
 you edited the exported image), osxphotos will detect this as a difference and
 re-export the original image from the library thus overwriting the changes.
 If using --update, the exported library should be treated as a backup, not a
-working copy where you intend to make changes.
+working copy where you intend to make changes. If you do edit or process the
+exported files and do not want them to be overwritten withsubsequent --update,
+use --ignore-signature which will match filename but not file signature when
+exporting.
 
 Note: The number of files reported for export and the number actually exported
 may differ due to live photos, associated raw images, and edited photos which

osxphotos/__main__.py

@@ -146,6 +146,9 @@ class ExportCommand(click.Command):
             + "exported image), osxphotos will detect this as a difference and re-export the original image "
             + "from the library thus overwriting the changes.  If using --update, the exported library "
             + "should be treated as a backup, not a working copy where you intend to make changes. "
+            + "If you do edit or process the exported files and do not want them to be overwritten with"
+            + "subsequent --update, use --ignore-signature which will match filename but not file signature when "
+            + "exporting."
         )
         formatter.write("\n")
         formatter.write_text(
@@ -1218,6 +1221,15 @@ def query(
     is_flag=True,
     help="Only export new or updated files. See notes below on export and --update.",
 )
+@click.option(
+    "--ignore-signature",
+    is_flag=True,
+    help="When used with --update, ignores file signature when updating files. "
+    "This is useful if you have processed or edited exported photos changing the "
+    "file signature (size & modification date). In this case, --update would normally "
+    "re-export the processed files but with --ignore-signature, files which exist "
+    "in the export directory will not be re-exported.",
+)
 @click.option(
     "--dry-run",
     is_flag=True,
@@ -1496,6 +1508,7 @@ def export(
     verbose,
     missing,
     update,
+    ignore_signature,
     dry_run,
     export_as_hardlink,
     touch_file,
@@ -1621,6 +1634,7 @@ def export(
         verbose = cfg.verbose
         missing = cfg.missing
         update = cfg.update
+        ignore_signature = cfg.ignore_signature
         dry_run = cfg.dry_run
         export_as_hardlink = cfg.export_as_hardlink
         touch_file = cfg.touch_file
@@ -1714,6 +1728,7 @@ def export(
     dependent_options = [
         ("missing", ("download_missing", "use_photos_export")),
         ("jpeg_quality", ("convert_to_jpeg")),
+        ("ignore_signature", ("update")),
     ]
     try:
         cfg.validate(exclusive=exclusive_options, dependent=dependent_options, cli=True)
@@ -1932,6 +1947,7 @@ def export(
                     export_by_date=export_by_date,
                     sidecar=sidecar,
                     update=update,
+                    ignore_signature=ignore_signature,
                     export_as_hardlink=export_as_hardlink,
                     overwrite=overwrite,
                     export_edited=export_edited,
@@ -1990,6 +2006,7 @@ def export(
                         export_by_date=export_by_date,
                         sidecar=sidecar,
                         update=update,
+                        ignore_signature=ignore_signature,
                         export_as_hardlink=export_as_hardlink,
                         overwrite=overwrite,
                         export_edited=export_edited,
@@ -2560,6 +2577,7 @@ def export_photo(
     export_by_date=None,
     sidecar=None,
     update=None,
+    ignore_signature=None,
     export_as_hardlink=None,
     overwrite=None,
     export_edited=None,
@@ -2766,6 +2784,7 @@ def export_photo(
                             keyword_template=keyword_template,
                             description_template=description_template,
                             update=update,
+                            ignore_signature=ignore_signature,
                             export_db=export_db,
                             fileutil=fileutil,
                             dry_run=dry_run,
@@ -2872,6 +2891,7 @@ def export_photo(
                             keyword_template=keyword_template,
                             description_template=description_template,
                             update=update,
+                            ignore_signature=ignore_signature,
                             export_db=export_db,
                             fileutil=fileutil,
                             dry_run=dry_run,

osxphotos/_version.py

@@ -1,5 +1,5 @@
 """ version info """
 
-__version__ = "0.38.4"
+__version__ = "0.38.5"
 
 

osxphotos/photoinfo/_photoinfo_export.py

@@ -285,7 +285,8 @@ def export(
     when exporting metadata with exiftool or sidecar
     keyword_template: (list of strings); list of template strings that will be rendered as used as keywords
     description_template: string; optional template string that will be rendered for use as photo description
-    returns: list of photos exported
+    
+    Returns: list of photos exported
     """
 
     # Implementation note: calls export2 to actually do the work
@@ -333,6 +334,7 @@ def export2(
     keyword_template=None,
     description_template=None,
     update=False,
+    ignore_signature=False,
     export_db=None,
     fileutil=FileUtil,
     dry_run=False,
@@ -376,6 +378,7 @@ def export2(
     description_template: string; optional template string that will be rendered for use as photo description
     update: (boolean, default=False); if True export will run in update mode, that is, it will
             not export the photo if the current version already exists in the destination
+    ignore_signature: (bool, default=False), ignore file signature when used with update (look only at filename)
     export_db: (ExportDB_ABC); instance of a class that conforms to ExportDB_ABC with methods
             for getting/setting data related to exported files to compare update state
     fileutil: (FileUtilABC); class that conforms to FileUtilABC with various file utilities
@@ -606,6 +609,7 @@ def export2(
             fileutil=fileutil,
             edited=edited,
             jpeg_quality=jpeg_quality,
+            ignore_signature=ignore_signature,
         )
         exported_files = results.exported
         update_new_files = results.new
@@ -631,6 +635,7 @@ def export2(
                     touch_file,
                     False,
                     fileutil=fileutil,
+                    ignore_signature=ignore_signature,
                 )
                 exported_files.extend(results.exported)
                 update_new_files.extend(results.new)
@@ -657,6 +662,7 @@ def export2(
                     convert_to_jpeg,
                     fileutil=fileutil,
                     jpeg_quality=jpeg_quality,
+                    ignore_signature=ignore_signature,
                 )
                 exported_files.extend(results.exported)
                 update_new_files.extend(results.new)
@@ -963,6 +969,7 @@ def _export_photo(
     fileutil=FileUtil,
     edited=False,
     jpeg_quality=1.0,
+    ignore_signature=None,
 ):
     """Helper function for export()
         Does the actual copy or hardlink taking the appropriate
@@ -983,6 +990,7 @@ def _export_photo(
         fileutil: FileUtil class that conforms to fileutil.FileUtilABC
         edited: bool; set to True if exporting edited version of photo
         jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0.  A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
+        ignore_signature: bool, ignore file signature when used with update (look only at filename)
 
     Returns:
         ExportResults
@@ -1008,7 +1016,10 @@ def _export_photo(
         cmp_touch, cmp_orig = False, False
         if dest_exists:
             # update, destination exists, but we might not need to replace it...
-            if exiftool:
+            if ignore_signature:
+                cmp_orig = True
+                cmp_touch = fileutil.cmp(src, dest, mtime1=int(self.date.timestamp()))
+            elif exiftool:
                 sig_exif = export_db.get_stat_exif_for_file(dest_str)
                 cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif)
                 sig_exif = (sig_exif[0], sig_exif[1], int(self.date.timestamp()))

tests/Test-10.15.7.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
 	<key>pid</key>
-	<integer>464</integer>
+	<integer>485</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:43Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:42Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:43Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:43Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:41Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:43Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2020-10-17T23:45:33Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2020-10-17T23:45:24Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2020-10-17T23:45:26Z</date>
+	<date>2020-12-16T05:41:44Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2020-10-17T23:45:25Z</date>
+	<date>2020-12-16T05:41:43Z</date>
 </dict>
 </plist>

tests/test_catalina_10_15_7.py

@@ -21,7 +21,7 @@ PHOTOS_LIBRARY_PATH = "/Test-10.15.7.photoslibrary"
 PHOTOS_DB_LEN = 18
 PHOTOS_NOT_IN_TRASH_LEN = 16
 PHOTOS_IN_TRASH_LEN = 2
-PHOTOS_DB_IMPORT_SESSIONS = 12
+PHOTOS_DB_IMPORT_SESSIONS = 13
 
 KEYWORDS = [
     "Kids",
@@ -93,7 +93,7 @@ UUID_DICT = {
     "not_intrash": "DC99FBDD-7A52-4100-A5BB-344131646C30",
     "intrash_person_keywords": "6FD38366-3BF2-407D-81FE-7153EB6125B6",
     "import_session": "8846E3E6-8AC8-4857-8448-E3D025784410",
-    "movie": "2CE332F2-D578-4769-AEFA-7631BB77AA41",
+    "movie": "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3",
 }
 
 UUID_PUMPKIN_FARM = [

tests/test_cli.py

@@ -59,6 +59,8 @@ CLI_EXPORT_FILENAMES = [
     "wedding_edited.jpeg",
 ]
 
+CLI_EXPORT_IGNORE_SIGNATURE_FILENAMES = ["Tulips.jpg", "wedding.jpg"]
+
 CLI_EXPORT_FILENAMES_ALBUM = ["Pumkins1.jpg", "Pumkins2.jpg", "Pumpkins3.jpg"]
 
 CLI_EXPORT_FILENAMES_ALBUM_UNICODE = ["IMG_4547.jpg"]
@@ -372,10 +374,10 @@ CLI_EXIFTOOL_QUICKTIME = {
         "QuickTime:CreateDate": "2020:01:05 22:13:13",
         "QuickTime:ModifyDate": "2020:01:05 22:13:13",
     },
-    "2CE332F2-D578-4769-AEFA-7631BB77AA41": {
-        "File:FileName": "Jellyfish.mp4",
+    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3": {
+        "File:FileName": "Jellyfish1.mp4",
         "XMP:Description": "Jellyfish Video",
-        "XMP:Title": "Jellyfish",
+        "XMP:Title": "Jellyfish1",
         "XMP:TagsList": "Travel",
         "XMP:Subject": "Travel",
         "QuickTime:GPSCoordinates": "34.053345 -118.242349",
@@ -516,6 +518,12 @@ UUID_NO_LIKES = [
 ]
 
 
+def modify_file(filename):
+    """ appends data to a file to modify it """
+    with open(filename, "ab") as fd:
+        fd.write(b"foo")
+
+
 @pytest.fixture(autouse=True)
 def reset_globals():
     """ reset globals in __main__ that tests may have changed """
@@ -3787,6 +3795,54 @@ def test_export_touch_files_exiftool_update():
         assert "skipped: 18" in result.output
 
 
+def test_export_ignore_signature():
+    """ test export with --ignore-signature """
+    from osxphotos.__main__ import export
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        # first, export some files
+        result = runner.invoke(export, [os.path.join(cwd, PHOTOS_DB_15_7), ".", "-V"])
+        assert result.exit_code == 0
+
+        # modify a couple of files
+        for filename in CLI_EXPORT_IGNORE_SIGNATURE_FILENAMES:
+            modify_file(f"./{filename}")
+
+        # export with --update and --ignore-signature
+        # which should ignore the two modified files
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                ".",
+                "-V",
+                "--update",
+                "--ignore-signature",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "exported: 0, updated: 0" in result.output
+
+        # export with --update and not --ignore-signature
+        # which should updated the two modified files
+        result = runner.invoke(
+            export, [os.path.join(cwd, PHOTOS_DB_15_7), ".", "-V", "--update"]
+        )
+        assert result.exit_code == 0
+        assert "updated: 2" in result.output
+
+        # run --update again, should be 0 files exported
+        result = runner.invoke(
+            export, [os.path.join(cwd, PHOTOS_DB_15_7), ".", "-V", "--update"]
+        )
+        assert result.exit_code == 0
+        assert "exported: 0, updated: 0" in result.output
+
+
 def test_labels():
     """Test osxphotos labels """
     import json