Added --force-update, #621

CHANGELOG.md

@@ -4,6 +4,39 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.45.8](https://github.com/RhetTbull/osxphotos/compare/v0.45.6...v0.45.8)
+
+> 5 February 2022
+
+- Fixed exiftool to ignore unsupported file types, #615 [`1ae6270`](https://github.com/RhetTbull/osxphotos/commit/1ae627056113fc4655f1b24cfbbdf0efc04489e7)
+- Updated tests [`55a601c`](https://github.com/RhetTbull/osxphotos/commit/55a601c07ea1384623c55d5c1d26b568df5d7823)
+- Additional fix for #615 [`1d6bc4e`](https://github.com/RhetTbull/osxphotos/commit/1d6bc4e09e3c2359a21f842fadd781920606812e)
+
+#### [v0.45.6](https://github.com/RhetTbull/osxphotos/compare/v0.45.5...v0.45.6)
+
+> 5 February 2022
+
+- Fix for unicode in query strings, #618 [`9b247ac`](https://github.com/RhetTbull/osxphotos/commit/9b247acd1cc4b2def59fdd18a6fb3c8eb9914f11)
+- Fix for --name searching only original_filename on Photos 5+, #594 [`cd02144`](https://github.com/RhetTbull/osxphotos/commit/cd02144ac33cc1c13a20358133971c84d35b8a57)
+
+#### [v0.45.5](https://github.com/RhetTbull/osxphotos/compare/v0.45.4...v0.45.5)
+
+> 5 February 2022
+
+- Fix for #561, no really, I mean it this time [`b3d3e14`](https://github.com/RhetTbull/osxphotos/commit/b3d3e14ffe41fbb22edb614b24f3985f379766a2)
+- Updated docs [skip ci] [`2b9ea11`](https://github.com/RhetTbull/osxphotos/commit/2b9ea11701799af9a661a8e2af70fca97235f487)
+- Updated tests for #561 [skip ci] [`77a49a0`](https://github.com/RhetTbull/osxphotos/commit/77a49a09a1bee74113a7114c543fbc25fa410ffc)
+
+#### [v0.45.4](https://github.com/RhetTbull/osxphotos/compare/v0.45.3...v0.45.4)
+
+> 3 February 2022
+
+- docs: add oPromessa as a contributor for ideas, test [`#611`](https://github.com/RhetTbull/osxphotos/pull/611)
+- Fix for filenames with special characters, #561, #618 [`f3063d3`](https://github.com/RhetTbull/osxphotos/commit/f3063d35be3c96342d83dbd87ddd614a2001bff4)
+- Updated docs [skip ci] [`06c5bbf`](https://github.com/RhetTbull/osxphotos/commit/06c5bbfcfdf591a4a5d43f1456adaa27385fe01a)
+- Added progress counter, #601 [`7ab5007`](https://github.com/RhetTbull/osxphotos/commit/7ab500740b28594dcd778140e10991f839220e9d)
+- Updated known issues [skip ci] [`e32090b`](https://github.com/RhetTbull/osxphotos/commit/e32090bf39cb786171b49443f878ffdbab774420)
+
 #### [v0.45.3](https://github.com/RhetTbull/osxphotos/compare/v0.45.2...v0.45.3)
 
 > 29 January 2022

MANIFEST.in

@@ -1,7 +1,7 @@
-include README.md
+include osxphotos/*.json
-include README.rst
+include osxphotos/*.md
-include osxphotos/templates/*
 include osxphotos/phototemplate.tx
-include osxphotos/phototemplate.md
+include osxphotos/queries/*
-include osxphotos/tutorial.md
+include osxphotos/templates/*
-include osxphotos/queries/*
+include README.md
+include README.rst

README.md

@@ -783,8 +783,15 @@ Options:
                                   folder.
   --deleted-only                  Include only photos from the 'Recently
                                   Deleted' folder.
-  --update                        Only export new or updated files. See notes
+  --update                        Only export new or updated files. See also
-                                  below on export and --update.
+                                  --force-update and notes below on export and
+                                  --update.
+  --force-update                  Only export new or updated files. Unlike
+                                  --update, --force-update will re-export photos
+                                  if their metadata has changed even if this
+                                  would not otherwise trigger an export. See
+                                  also --update and notes below on export and
+                                  --update.
   --ignore-signature              When used with '--update', ignores file
                                   signature when updating files. This is useful
                                   if you have processed or edited exported
@@ -1725,7 +1732,7 @@ Substitution                    Description
 {lf}                            A line feed: '\n', alias for {newline}
 {cr}                            A carriage return: '\r'
 {crlf}                          a carriage return + line feed: '\r\n'
-{osxphotos_version}             The osxphotos version, e.g. '0.45.4'
+{osxphotos_version}             The osxphotos version, e.g. '0.45.9'
 {osxphotos_cmd_line}            The full command line used to run osxphotos
 
 The following substitutions may result in multiple values. Thus if specified for
@@ -3629,7 +3636,7 @@ The following template field substitutions are availabe for use the templating s
 |{lf}|A line feed: '\n', alias for {newline}|
 |{cr}|A carriage return: '\r'|
 |{crlf}|a carriage return + line feed: '\r\n'|
-|{osxphotos_version}|The osxphotos version, e.g. '0.45.4'|
+|{osxphotos_version}|The osxphotos version, e.g. '0.45.9'|
 |{osxphotos_cmd_line}|The full command line used to run osxphotos|
 |{album}|Album(s) photo is contained in|
 |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 001a184f6f166bf8f64bf9bb56e7b73e
+config: a320d2e66b198895ef0b12b1f5934727
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '0.45.4',
+    VERSION: '0.45.9',
     LANGUAGE: 'None',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

docs/cli.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/genindex.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Index &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Index &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/index.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/modules.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/reference.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos package &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos package &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/search.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Search &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Search &#8212; osxphotos 0.45.9 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     

osxphotos.spec

@@ -14,6 +14,7 @@ datas = [
     ("osxphotos/phototemplate.tx", "osxphotos"),
     ("osxphotos/phototemplate.md", "osxphotos"),
     ("osxphotos/tutorial.md", "osxphotos"),
+    ("osxphotos/exiftool_filetypes.json", "osxphotos"),
 ]
 package_imports = [["photoscript", ["photoscript.applescript"]]]
 for package, files in package_imports:

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.45.4"
+__version__ = "0.45.9"

osxphotos/cli.py

@@ -691,7 +691,15 @@ def cli(ctx, db, json_, debug):
 @click.option(
     "--update",
     is_flag=True,
-    help="Only export new or updated files. See notes below on export and --update.",
+    help="Only export new or updated files. "
+    "See also --force-update and notes below on export and --update.",
+)
+@click.option(
+    "--force-update",
+    is_flag=True,
+    help="Only export new or updated files. Unlike --update, --force-update will re-export photos "
+    "if their metadata has changed even if this would not otherwise trigger an export. "
+    "See also --update and notes below on export and --update.",
 )
 @click.option(
     "--ignore-signature",
@@ -1235,6 +1243,7 @@ def export(
     timestamp,
     missing,
     update,
+    force_update,
     ignore_signature,
     only_new,
     dry_run,
@@ -1398,133 +1407,134 @@ def export(
 
         # re-set the local vars to the corresponding config value
         # this isn't elegant but avoids having to rewrite this function to use cfg.varname for every parameter
-        db = cfg.db
+        add_exported_to_album = cfg.add_exported_to_album
-        photos_library = cfg.photos_library
+        add_missing_to_album = cfg.add_missing_to_album
-        keyword = cfg.keyword
+        add_skipped_to_album = cfg.add_skipped_to_album
-        person = cfg.person
         album = cfg.album
-        folder = cfg.folder
-        name = cfg.name
-        uuid = cfg.uuid
-        uuid_from_file = cfg.uuid_from_file
-        title = cfg.title
-        no_title = cfg.no_title
-        description = cfg.description
-        no_description = cfg.no_description
-        uti = cfg.uti
-        ignore_case = cfg.ignore_case
-        edited = cfg.edited
-        external_edit = cfg.external_edit
-        favorite = cfg.favorite
-        not_favorite = cfg.not_favorite
-        hidden = cfg.hidden
-        not_hidden = cfg.not_hidden
-        shared = cfg.shared
-        not_shared = cfg.not_shared
-        from_date = cfg.from_date
-        to_date = cfg.to_date
-        from_time = cfg.from_time
-        to_time = cfg.to_time
-        verbose = cfg.verbose
-        missing = cfg.missing
-        update = cfg.update
-        ignore_signature = cfg.ignore_signature
-        dry_run = cfg.dry_run
-        export_as_hardlink = cfg.export_as_hardlink
-        touch_file = cfg.touch_file
-        overwrite = cfg.overwrite
-        retry = cfg.retry
-        export_by_date = cfg.export_by_date
-        skip_edited = cfg.skip_edited
-        skip_original_if_edited = cfg.skip_original_if_edited
-        skip_bursts = cfg.skip_bursts
-        skip_live = cfg.skip_live
-        skip_raw = cfg.skip_raw
-        skip_uuid = cfg.skip_uuid
-        skip_uuid_from_file = cfg.skip_uuid_from_file
-        person_keyword = cfg.person_keyword
         album_keyword = cfg.album_keyword
-        keyword_template = cfg.keyword_template
+        beta = cfg.beta
-        replace_keywords = cfg.replace_keywords
-        description_template = cfg.description_template
-        finder_tag_template = cfg.finder_tag_template
-        finder_tag_keywords = cfg.finder_tag_keywords
-        xattr_template = cfg.xattr_template
-        current_name = cfg.current_name
-        convert_to_jpeg = cfg.convert_to_jpeg
-        jpeg_quality = cfg.jpeg_quality
-        sidecar = cfg.sidecar
-        sidecar_drop_ext = cfg.sidecar_drop_ext
-        only_photos = cfg.only_photos
-        only_movies = cfg.only_movies
         burst = cfg.burst
-        not_burst = cfg.not_burst
+        cleanup = cfg.cleanup
-        live = cfg.live
+        convert_to_jpeg = cfg.convert_to_jpeg
-        not_live = cfg.not_live
+        current_name = cfg.current_name
-        download_missing = cfg.download_missing
+        db = cfg.db
-        exiftool = cfg.exiftool
-        exiftool_path = cfg.exiftool_path
-        exiftool_option = cfg.exiftool_option
-        exiftool_merge_keywords = cfg.exiftool_merge_keywords
-        exiftool_merge_persons = cfg.exiftool_merge_persons
-        ignore_date_modified = cfg.ignore_date_modified
-        portrait = cfg.portrait
-        not_portrait = cfg.not_portrait
-        screenshot = cfg.screenshot
-        not_screenshot = cfg.not_screenshot
-        slow_mo = cfg.slow_mo
-        not_slow_mo = cfg.not_slow_mo
-        time_lapse = cfg.time_lapse
-        not_time_lapse = cfg.not_time_lapse
-        hdr = cfg.hdr
-        not_hdr = cfg.not_hdr
-        selfie = cfg.selfie
-        not_selfie = cfg.not_selfie
-        panorama = cfg.panorama
-        not_panorama = cfg.not_panorama
-        has_raw = cfg.has_raw
-        directory = cfg.directory
-        filename_template = cfg.filename_template
-        jpeg_ext = cfg.jpeg_ext
-        strip = cfg.strip
-        edited_suffix = cfg.edited_suffix
-        original_suffix = cfg.original_suffix
-        place = cfg.place
-        no_place = cfg.no_place
-        location = cfg.location
-        no_location = cfg.no_location
-        has_comment = cfg.has_comment
-        no_comment = cfg.no_comment
-        has_likes = cfg.has_likes
-        no_likes = cfg.no_likes
-        label = cfg.label
         deleted = cfg.deleted
         deleted_only = cfg.deleted_only
-        use_photos_export = cfg.use_photos_export
+        description = cfg.description
-        use_photokit = cfg.use_photokit
+        description_template = cfg.description_template
-        report = cfg.report
+        directory = cfg.directory
-        cleanup = cfg.cleanup
+        download_missing = cfg.download_missing
-        add_exported_to_album = cfg.add_exported_to_album
+        dry_run = cfg.dry_run
-        add_skipped_to_album = cfg.add_skipped_to_album
-        add_missing_to_album = cfg.add_missing_to_album
-        exportdb = cfg.exportdb
-        beta = cfg.beta
-        only_new = cfg.only_new
-        in_album = cfg.in_album
-        not_in_album = cfg.not_in_album
-        min_size = cfg.min_size
-        max_size = cfg.max_size
-        regex = cfg.regex
-        selected = cfg.selected
-        exif = cfg.exif
-        query_eval = cfg.query_eval
-        query_function = cfg.query_function
         duplicate = cfg.duplicate
+        edited = cfg.edited
+        edited_suffix = cfg.edited_suffix
+        exif = cfg.exif
+        exiftool = cfg.exiftool
+        exiftool_merge_keywords = cfg.exiftool_merge_keywords
+        exiftool_merge_persons = cfg.exiftool_merge_persons
+        exiftool_option = cfg.exiftool_option
+        exiftool_path = cfg.exiftool_path
+        export_as_hardlink = cfg.export_as_hardlink
+        export_by_date = cfg.export_by_date
+        exportdb = cfg.exportdb
+        external_edit = cfg.external_edit
+        favorite = cfg.favorite
+        filename_template = cfg.filename_template
+        finder_tag_keywords = cfg.finder_tag_keywords
+        finder_tag_template = cfg.finder_tag_template
+        folder = cfg.folder
+        force_update = cfg.force_update
+        from_date = cfg.from_date
+        from_time = cfg.from_time
+        has_comment = cfg.has_comment
+        has_likes = cfg.has_likes
+        has_raw = cfg.has_raw
+        hdr = cfg.hdr
+        hidden = cfg.hidden
+        ignore_case = cfg.ignore_case
+        ignore_date_modified = cfg.ignore_date_modified
+        ignore_signature = cfg.ignore_signature
+        in_album = cfg.in_album
+        jpeg_ext = cfg.jpeg_ext
+        jpeg_quality = cfg.jpeg_quality
+        keyword = cfg.keyword
+        keyword_template = cfg.keyword_template
+        label = cfg.label
+        live = cfg.live
+        location = cfg.location
+        max_size = cfg.max_size
+        min_size = cfg.min_size
+        missing = cfg.missing
+        name = cfg.name
+        no_comment = cfg.no_comment
+        no_description = cfg.no_description
+        no_likes = cfg.no_likes
+        no_location = cfg.no_location
+        no_place = cfg.no_place
+        no_title = cfg.no_title
+        not_burst = cfg.not_burst
+        not_favorite = cfg.not_favorite
+        not_hdr = cfg.not_hdr
+        not_hidden = cfg.not_hidden
+        not_in_album = cfg.not_in_album
+        not_live = cfg.not_live
+        not_panorama = cfg.not_panorama
+        not_portrait = cfg.not_portrait
+        not_screenshot = cfg.not_screenshot
+        not_selfie = cfg.not_selfie
+        not_shared = cfg.not_shared
+        not_slow_mo = cfg.not_slow_mo
+        not_time_lapse = cfg.not_time_lapse
+        only_movies = cfg.only_movies
+        only_new = cfg.only_new
+        only_photos = cfg.only_photos
+        original_suffix = cfg.original_suffix
+        overwrite = cfg.overwrite
+        panorama = cfg.panorama
+        person = cfg.person
+        person_keyword = cfg.person_keyword
+        photos_library = cfg.photos_library
+        place = cfg.place
+        portrait = cfg.portrait
         post_command = cfg.post_command
         post_function = cfg.post_function
         preview = cfg.preview
-        preview_suffix = cfg.preview_suffix
         preview_if_missing = cfg.preview_if_missing
+        preview_suffix = cfg.preview_suffix
+        query_eval = cfg.query_eval
+        query_function = cfg.query_function
+        regex = cfg.regex
+        replace_keywords = cfg.replace_keywords
+        report = cfg.report
+        retry = cfg.retry
+        screenshot = cfg.screenshot
+        selected = cfg.selected
+        selfie = cfg.selfie
+        shared = cfg.shared
+        sidecar = cfg.sidecar
+        sidecar_drop_ext = cfg.sidecar_drop_ext
+        skip_bursts = cfg.skip_bursts
+        skip_edited = cfg.skip_edited
+        skip_live = cfg.skip_live
+        skip_original_if_edited = cfg.skip_original_if_edited
+        skip_raw = cfg.skip_raw
+        skip_uuid = cfg.skip_uuid
+        skip_uuid_from_file = cfg.skip_uuid_from_file
+        slow_mo = cfg.slow_mo
+        strip = cfg.strip
+        time_lapse = cfg.time_lapse
+        title = cfg.title
+        to_date = cfg.to_date
+        to_time = cfg.to_time
+        touch_file = cfg.touch_file
+        update = cfg.update
+        use_photokit = cfg.use_photokit
+        use_photos_export = cfg.use_photos_export
+        uti = cfg.uti
+        uuid = cfg.uuid
+        uuid_from_file = cfg.uuid_from_file
+        verbose = cfg.verbose
+        xattr_template = cfg.xattr_template
 
         # config file might have changed verbose
         VERBOSE = bool(verbose)
@@ -1564,8 +1574,8 @@ def export(
     dependent_options = [
         ("missing", ("download_missing", "use_photos_export")),
         ("jpeg_quality", ("convert_to_jpeg")),
-        ("ignore_signature", ("update")),
+        ("ignore_signature", ("update", "force_update")),
-        ("only_new", ("update")),
+        ("only_new", ("update", "force_update")),
         ("exiftool_option", ("exiftool")),
         ("exiftool_merge_keywords", ("exiftool", "sidecar")),
         ("exiftool_merge_persons", ("exiftool", "sidecar")),
@@ -1905,51 +1915,52 @@ def export(
                 export_results = export_photo(
                     photo=p,
                     dest=dest,
-                    verbose=verbose,
+                    album_keyword=album_keyword,
-                    export_by_date=export_by_date,
+                    convert_to_jpeg=convert_to_jpeg,
-                    sidecar=sidecar,
+                    description_template=description_template,
-                    sidecar_drop_ext=sidecar_drop_ext,
+                    directory=directory,
-                    update=update,
-                    ignore_signature=ignore_signature,
-                    export_as_hardlink=export_as_hardlink,
-                    overwrite=overwrite,
-                    export_edited=export_edited,
-                    skip_original_if_edited=skip_original_if_edited,
-                    original_name=original_name,
-                    export_live=export_live,
                     download_missing=download_missing,
-                    exiftool=exiftool,
+                    dry_run=dry_run,
+                    edited_suffix=edited_suffix,
                     exiftool_merge_keywords=exiftool_merge_keywords,
                     exiftool_merge_persons=exiftool_merge_persons,
-                    directory=directory,
-                    filename_template=filename_template,
-                    export_raw=export_raw,
-                    album_keyword=album_keyword,
-                    person_keyword=person_keyword,
-                    keyword_template=keyword_template,
-                    description_template=description_template,
-                    export_db=export_db,
-                    fileutil=fileutil,
-                    dry_run=dry_run,
-                    touch_file=touch_file,
-                    edited_suffix=edited_suffix,
-                    original_suffix=original_suffix,
-                    use_photos_export=use_photos_export,
-                    convert_to_jpeg=convert_to_jpeg,
-                    jpeg_quality=jpeg_quality,
-                    ignore_date_modified=ignore_date_modified,
-                    use_photokit=use_photokit,
                     exiftool_option=exiftool_option,
-                    strip=strip,
+                    exiftool=exiftool,
+                    export_as_hardlink=export_as_hardlink,
+                    export_by_date=export_by_date,
+                    export_db=export_db,
+                    export_dir=dest,
+                    export_edited=export_edited,
+                    export_live=export_live,
+                    export_preview=preview,
+                    export_raw=export_raw,
+                    filename_template=filename_template,
+                    fileutil=fileutil,
+                    force_update=force_update,
+                    ignore_date_modified=ignore_date_modified,
+                    ignore_signature=ignore_signature,
                     jpeg_ext=jpeg_ext,
+                    jpeg_quality=jpeg_quality,
+                    keyword_template=keyword_template,
+                    num_photos=num_photos,
+                    original_name=original_name,
+                    original_suffix=original_suffix,
+                    overwrite=overwrite,
+                    person_keyword=person_keyword,
+                    photo_num=photo_num,
+                    preview_if_missing=preview_if_missing,
+                    preview_suffix=preview_suffix,
                     replace_keywords=replace_keywords,
                     retry=retry,
-                    export_dir=dest,
+                    sidecar_drop_ext=sidecar_drop_ext,
-                    export_preview=preview,
+                    sidecar=sidecar,
-                    preview_suffix=preview_suffix,
+                    skip_original_if_edited=skip_original_if_edited,
-                    preview_if_missing=preview_if_missing,
+                    strip=strip,
-                    photo_num=photo_num,
+                    touch_file=touch_file,
-                    num_photos=num_photos,
+                    update=update,
+                    use_photokit=use_photokit,
+                    use_photos_export=use_photos_export,
+                    verbose=verbose,
                 )
 
                 if post_function:
@@ -2062,7 +2073,7 @@ def export(
             fp.close()
 
         photo_str_total = "photos" if len(photos) != 1 else "photo"
-        if update:
+        if update or force_update:
             summary = (
                 f"Processed: {len(photos)} {photo_str_total}, "
                 f"exported: {len(results.new)}, "
@@ -2087,6 +2098,8 @@ def export(
 
     # cleanup files and do report if needed
     if cleanup:
+        db_file = str(pathlib.Path(export_db_path).resolve())
+        db_files = [db_file, db_file + "-wal", db_file + "-shm"]
         all_files = (
             results.exported
             + results.skipped
@@ -2105,7 +2118,7 @@ def export(
             + results.missing
             # include files that have error in case they exist from previous export
             + [r[0] for r in results.error]
-            + [str(pathlib.Path(export_db_path).resolve())]
+            + db_files
         )
         click.echo(f"Cleaning up {dest}")
         cleaned_files, cleaned_dirs = cleanup_files(dest, all_files, fileutil)
@@ -2590,6 +2603,7 @@ def export_photo(
     sidecar=None,
     sidecar_drop_ext=False,
     update=None,
+    force_update=None,
     ignore_signature=None,
     export_as_hardlink=None,
     overwrite=None,
@@ -2636,47 +2650,47 @@ def export_photo(
     Args:
         photo: PhotoInfo object
         dest: destination path as string
-        verbose: boolean; print verbose output
+        album_keyword: bool; if True, exports album names as keywords in metadata
-        export_by_date: boolean; create export folder in form dest/YYYY/MM/DD
+        convert_to_jpeg: bool; if True, converts non-jpeg images to jpeg
-        sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export
+        description_template: str; optional template string that will be rendered for use as photo description
-        sidecar_drop_ext: boolean; if True, drops photo extension from sidecar name
-        export_as_hardlink: boolean; hardlink files instead of copying them
-        overwrite: boolean; overwrite dest file if it already exists
-        original_name: boolean; use original filename instead of current filename
-        export_live: boolean; also export live video component if photo is a live photo
-                     live video will have same name as photo but with .mov extension
-        download_missing: attempt download of missing iCloud photos
-        exiftool: use exiftool to write EXIF metadata directly to exported photo
         directory: template used to determine output directory
-        filename_template: template use to determine output file
+        download_missing: attempt download of missing iCloud photos
-        export_raw: boolean; if True exports raw image associate with the photo
+        dry_run: bool; if True, doesn't actually export or update any files
-        export_edited: boolean; if True exports edited version of photo if there is one
+        exiftool_merge_keywords: bool; if True, merged keywords found in file's exif data (requires exiftool)
-        skip_original_if_edited: boolean; if True does not export original if photo has been edited
+        exiftool_merge_persons: bool; if True, merged persons found in file's exif data (requires exiftool)
-        album_keyword: boolean; if True, exports album names as keywords in metadata
-        person_keyword: boolean; if True, exports person names as keywords in metadata
-        keyword_template: list of strings; if provided use rendered template strings as keywords
-        description_template: string; optional template string that will be rendered for use as photo description
-        export_db: export database instance compatible with ExportDB_ABC
-        fileutil: file util class compatible with FileUtilABC
-        dry_run: boolean; if True, doesn't actually export or update any files
-        touch_file: boolean; sets file's modification time to match photo date
-        use_photos_export: boolean; if True forces the use of AppleScript to export even if photo not missing
-        convert_to_jpeg: boolean; if True, converts non-jpeg images to jpeg
-        jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0.  A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
-        ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
         exiftool_option: optional list flags (e.g. ["-m", "-F"]) to pass to exiftool
-        exiftool_merge_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
+        exiftool: bool; use exiftool to write EXIF metadata directly to exported photo
-        exiftool_merge_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
+        export_as_hardlink: bool; hardlink files instead of copying them
+        export_by_date: bool; create export folder in form dest/YYYY/MM/DD
+        export_db: export database instance compatible with ExportDB_ABC
+        export_dir: top-level export directory for {export_dir} template
+        export_edited: bool; if True exports edited version of photo if there is one
+        export_live: bool; also export live video component if photo is a live photo; live video will have same name as photo but with .mov extension
+        export_preview: export the preview image generated by Photos
+        export_raw: bool; if True exports raw image associate with the photo
+        filename_template: template use to determine output file
+        fileutil: file util class compatible with FileUtilABC
+        force_update: bool, only export updated photos but trigger export even if only metadata has changed
+        ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
         jpeg_ext: if not None, specify the extension to use for all JPEG images on export
+        jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0.  A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
+        keyword_template: list of strings; if provided use rendered template strings as keywords
+        num_photos: int, total number of photos that will be exported
+        original_name: bool; use original filename instead of current filename
+        overwrite: bool; overwrite dest file if it already exists
+        person_keyword: bool; if True, exports person names as keywords in metadata
+        photo_num: int, which number photo in total of num_photos is being exported
+        preview_if_missing: bool, export preview if original is missing
+        preview_suffix: str, template to use as suffix for preview images
         replace_keywords: if True, --keyword-template replaces keywords instead of adding keywords
         retry: retry up to retry # of times if there's an error
-        export_dir: top-level export directory for {export_dir} template
+        sidecar_drop_ext: bool; if True, drops photo extension from sidecar name
-        export_preview: export the preview image generated by Photos
+        sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export
-        preview_suffix: str, template to use as suffix for preview images
+        skip_original_if_edited: bool; if True does not export original if photo has been edited
-        preview_if_missing: bool, export preview if original is missing
+        touch_file: bool; sets file's modification time to match photo date
-        photo_num: int, which number photo in total of num_photos is being exported
+        update: bool, only export updated photos
-        num_photos: int, total number of photos that will be exported
+        use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing
-
+        verbose: bool; print verbose output
     Returns:
         list of path(s) of exported photo or None if photo was missing
 
@@ -2822,6 +2836,7 @@ def export_photo(
                 export_raw=export_raw,
                 filename=original_filename,
                 fileutil=fileutil,
+                force_update=force_update,
                 ignore_date_modified=ignore_date_modified,
                 ignore_signature=ignore_signature,
                 jpeg_ext=jpeg_ext,
@@ -2934,6 +2949,7 @@ def export_photo(
                     export_raw=not export_original and export_raw,
                     filename=edited_filename,
                     fileutil=fileutil,
+                    force_update=force_update,
                     ignore_date_modified=ignore_date_modified,
                     ignore_signature=ignore_signature,
                     jpeg_ext=jpeg_ext,
@@ -3017,6 +3033,7 @@ def export_photo_to_directory(
     export_raw,
     filename,
     fileutil,
+    force_update,
     ignore_date_modified,
     ignore_signature,
     jpeg_ext,
@@ -3041,53 +3058,24 @@ def export_photo_to_directory(
     """Export photo to directory dest_path"""
 
     results = ExportResults()
-    # TODO: can be updated to let export do all the missing logic
+
-    if export_original:
+    # don't try to export photos in the trash if they're missing
-        if missing and not preview_if_missing:
+    photo_path = photo.path if export_original else photo.path_edited
-            space = " " if not verbose else ""
+    if photo.intrash and not photo_path and not preview_if_missing:
-            verbose_(
+        # skip deleted files if they're missing
-                f"{space}Skipping missing photo {photo.original_filename} ({photo.uuid})"
+        # as AppleScript/PhotoKit cannot export deleted photos
-            )
+        verbose_(
-            results.missing.append(str(pathlib.Path(dest_path) / filename))
+            f"Skipping missing deleted photo {photo.original_filename} ({photo.uuid})"
-        elif (
+        )
-            photo.intrash
+        results.missing.append(str(pathlib.Path(dest_path) / filename))
-            and (not photo.path or (download_missing or use_photos_export))
-            and not preview_if_missing
-        ):
-            # skip deleted files if they're missing or using use_photos_export
-            # as AppleScript/PhotoKit cannot export deleted photos
-            space = " " if not verbose else ""
-            verbose_(
-                f"{space}Skipping missing deleted photo {photo.original_filename} ({photo.uuid})"
-            )
-            results.missing.append(str(pathlib.Path(dest_path) / filename))
-            return results
-    elif not edited:
-        verbose_(f"Skipping original version of {photo.original_filename}")
         return results
-    else:
-        # exporting the edited version
-        if missing and not preview_if_missing:
-            space = " " if not verbose else ""
-            verbose_(f"{space}Skipping missing edited photo for {filename}")
-            results.missing.append(str(pathlib.Path(dest_path) / filename))
-            return results
-        elif (
-            photo.intrash
-            and (not photo.path_edited or (download_missing or use_photos_export))
-            and not preview_if_missing
-        ):
-            # skip deleted files if they're missing or using use_photos_export
-            # as AppleScript/PhotoKit cannot export deleted photos
-            space = " " if not verbose else ""
-            verbose_(
-                f"{space}Skipping missing deleted photo {photo.original_filename} ({photo.uuid})"
-            )
-            results.missing.append(str(pathlib.Path(dest_path) / filename))
-            return results
 
     render_options = RenderOptions(export_dir=export_dir, dest_path=dest_path)
 
+    if not export_original and not edited:
+        verbose_(f"Skipping original version of {photo.original_filename}")
+        return results
+
     tries = 0
     while tries <= retry:
         tries += 1
@@ -3104,6 +3092,7 @@ def export_photo_to_directory(
                 export_as_hardlink=export_as_hardlink,
                 export_db=export_db,
                 fileutil=fileutil,
+                force_update=force_update,
                 ignore_date_modified=ignore_date_modified,
                 ignore_signature=ignore_signature,
                 jpeg_ext=jpeg_ext,
@@ -3206,7 +3195,7 @@ def get_filenames_from_template(
     Args:
         photo: a PhotoInfo instance
         filename_template: a PhotoTemplate template string, may be None
-        original_name: boolean; if True, use photo's original filename instead of current filename
+        original_name: bool; if True, use photo's original filename instead of current filename
         dest_path: the path the photo will be exported to
         strip: if True, strips leading/trailing white space from resulting template
         edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version
@@ -3267,9 +3256,9 @@ def get_dirnames_from_template(
     Args:
         photo: a PhotoInstance object
         directory: a PhotoTemplate template string, may be None
-        export_by_date: boolean; if True, creates output directories in form YYYY-MM-DD
+        export_by_date: bool; if True, creates output directories in form YYYY-MM-DD
         dest: top-level destination directory
-        dry_run: boolean; if True, runs in dry-run mode and does not create output directories
+        dry_run: bool; if True, runs in dry-run mode and does not create output directories
         strip: if True, strips leading/trailing white space from resulting template
         edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version
 

osxphotos/exiftool.py

@@ -11,6 +11,7 @@ import html
 import json
 import logging
 import os
+import pathlib
 import re
 import shutil
 import subprocess
@@ -19,11 +20,12 @@ from functools import lru_cache  # pylint: disable=syntax-error
 
 __all__ = [
     "escape_str",
-    "unescape_str",
+    "exiftool_can_write",
-    "terminate_exiftool",
-    "get_exiftool_path",
     "ExifTool",
     "ExifToolCaching",
+    "get_exiftool_path",
+    "terminate_exiftool",
+    "unescape_str",
 ]
 
 # exiftool -stay_open commands outputs this EOF marker after command is run
@@ -33,6 +35,24 @@ EXIFTOOL_STAYOPEN_EOF_LEN = len(EXIFTOOL_STAYOPEN_EOF)
 # list of exiftool processes to cleanup when exiting or when terminate is called
 EXIFTOOL_PROCESSES = []
 
+# exiftool supported file types, created by utils/exiftool_supported_types.py
+EXIFTOOL_FILETYPES_JSON = "exiftool_filetypes.json"
+with (pathlib.Path(__file__).parent / EXIFTOOL_FILETYPES_JSON).open("r") as f:
+    EXIFTOOL_SUPPORTED_FILETYPES = json.load(f)
+
+
+def exiftool_can_write(suffix: str) -> bool:
+    """Return True if exiftool supports writing to a file with the given suffix, otherwise False"""
+    if not suffix:
+        return False
+    suffix = suffix.lower()
+    if suffix[0] == ".":
+        suffix = suffix[1:]
+    return (
+        suffix in EXIFTOOL_SUPPORTED_FILETYPES
+        and EXIFTOOL_SUPPORTED_FILETYPES[suffix]["write"]
+    )
+
 
 def escape_str(s):
     """escape string for use with exiftool -E"""

osxphotos/export_db.py

@@ -10,13 +10,18 @@ import sys
 from abc import ABC, abstractmethod
 from io import StringIO
 from sqlite3 import Error
+from typing import Union
 
 from ._constants import OSXPHOTOS_EXPORT_DB
 from ._version import __version__
+from .utils import normalize_fs_path
 
 __all__ = ["ExportDB_ABC", "ExportDBNoOp", "ExportDB", "ExportDBInMemory"]
 
-OSXPHOTOS_EXPORTDB_VERSION = "4.2"
+OSXPHOTOS_EXPORTDB_VERSION = "5.0"
+OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH = "4.3"
+OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES = "4.3"
+
 OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
 
 
@@ -99,6 +104,14 @@ class ExportDB_ABC(ABC):
     def set_detected_text_for_uuid(self, uuid, json_text):
         pass
 
+    @abstractmethod
+    def set_metadata_for_file(self, filename, metadata):
+        pass
+
+    @abstractmethod
+    def get_metadata_for_file(self, filename):
+        pass
+
     @abstractmethod
     def set_data(
         self,
@@ -110,6 +123,7 @@ class ExportDB_ABC(ABC):
         edited_stat=None,
         info_json=None,
         exif_json=None,
+        metadata=None,
     ):
         pass
 
@@ -179,6 +193,12 @@ class ExportDBNoOp(ExportDB_ABC):
     def set_detected_text_for_uuid(self, uuid, json_text):
         pass
 
+    def set_metadata_for_file(self, filename, metadata):
+        pass
+
+    def get_metadata_for_file(self, filename):
+        pass
+
     def set_data(
         self,
         filename,
@@ -189,6 +209,7 @@ class ExportDBNoOp(ExportDB_ABC):
         edited_stat=None,
         info_json=None,
         exif_json=None,
+        metadata=None,
     ):
         pass
 
@@ -211,12 +232,13 @@ class ExportDB(ExportDB_ABC):
         """query database for filename and return UUID
         returns None if filename not found in database
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filepath_normalized = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
             c.execute(
-                "SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,)
+                "SELECT uuid FROM files WHERE filepath_normalized = ?",
+                (filepath_normalized,),
             )
             results = c.fetchone()
             uuid = results[0] if results else None
@@ -228,7 +250,7 @@ class ExportDB(ExportDB_ABC):
     def set_uuid_for_file(self, filename, uuid):
         """set UUID of filename to uuid in the database"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -245,7 +267,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -266,7 +288,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -302,7 +324,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename (after exiftool has updated it)
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -323,7 +345,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename (after exiftool has updated it)
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -384,7 +406,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_exifdata_for_file(self, filename):
         """returns the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -402,7 +424,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_exifdata_for_file(self, filename, exifdata):
         """sets the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -416,7 +438,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_sidecar_for_file(self, filename):
         """returns the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -444,7 +466,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
         """sets the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -502,6 +524,39 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+    def set_metadata_for_file(self, filename, metadata):
+        """set metadata of filename in the database"""
+        filename = str(pathlib.Path(filename).relative_to(self._path))
+        filename_normalized = self._normalize_filepath(filename)
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "UPDATE files SET metadata = ? WHERE filepath_normalized = ?;",
+                (metadata, filename_normalized),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def get_metadata_for_file(self, filename):
+        """get metadata value for file"""
+        filename = self._normalize_filepath_relative(filename)
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT metadata FROM files WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            metadata = results[0] if results else None
+        except Error as e:
+            logging.warning(e)
+            metadata = None
+
+        return metadata
+
     def set_data(
         self,
         filename,
@@ -512,10 +567,11 @@ class ExportDB(ExportDB_ABC):
         edited_stat=None,
         info_json=None,
         exif_json=None,
+        metadata=None,
     ):
         """sets all the data for file and uuid at once; if any value is None, does not set it"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -565,6 +621,15 @@ class ExportDB(ExportDB_ABC):
                     "INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
                     (filename_normalized, exif_json),
                 )
+
+            if metadata is not None:
+                c.execute(
+                    "UPDATE files "
+                    + "SET metadata = ? "
+                    + "WHERE filepath_normalized = ?;",
+                    (metadata, filename_normalized),
+                )
+
             conn.commit()
         except Error as e:
             logging.warning(e)
@@ -577,7 +642,7 @@ class ExportDB(ExportDB_ABC):
             logging.warning(e)
 
     def _set_stat_for_file(self, table, filename, stats):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -590,7 +655,7 @@ class ExportDB(ExportDB_ABC):
         conn.commit()
 
     def _get_stat_for_file(self, table, filename):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         c = conn.cursor()
         c.execute(
@@ -617,7 +682,7 @@ class ExportDB(ExportDB_ABC):
             conn = self._get_db_connection(dbfile)
             if not conn:
                 raise Exception("Error getting connection to database {dbfile}")
-            self._create_db_tables(conn)
+            self._create_or_migrate_db_tables(conn)
             self.was_created = True
             self.was_upgraded = ()
         else:
@@ -625,11 +690,19 @@ class ExportDB(ExportDB_ABC):
             self.was_created = False
             version_info = self._get_database_version(conn)
             if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION:
-                self._create_db_tables(conn)
+                self._create_or_migrate_db_tables(conn)
                 self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
             else:
                 self.was_upgraded = ()
         self.version = OSXPHOTOS_EXPORTDB_VERSION
+
+        # turn on performance optimizations
+        c = conn.cursor()
+        c.execute("PRAGMA journal_mode=WAL;")
+        c.execute("PRAGMA synchronous=NORMAL;")
+        c.execute("PRAGMA cache_size=-100000;")
+        c.execute("PRAGMA temp_store=MEMORY;")
+
         return conn
 
     def _get_db_connection(self, dbfile):
@@ -649,104 +722,97 @@ class ExportDB(ExportDB_ABC):
         ).fetchone()
         return (version_info[0], version_info[1])
 
-    def _create_db_tables(self, conn):
+    def _create_or_migrate_db_tables(self, conn):
-        """create (if not already created) the necessary db tables for the export database
+        """create (if not already created) the necessary db tables for the export database and apply any needed migrations
-        conn: sqlite3 db connection
+
+        Args:
+            conn: sqlite3 db connection
         """
-        sql_commands = {
+        try:
-            "sql_version_table": """ CREATE TABLE IF NOT EXISTS version (
+            version = self._get_database_version(conn)
-                                id INTEGER PRIMARY KEY,
+        except Exception as e:
-                                osxphotos TEXT,
+            version = (__version__, OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES)
-                                exportdb TEXT 
+
-                                ); """,
+        # Current for version 4.3, for anything greater, do a migration after creation
-            "sql_about_table": """ CREATE TABLE IF NOT EXISTS about (
+        sql_commands = [
-                                id INTEGER PRIMARY KEY,
+            """ CREATE TABLE IF NOT EXISTS version (
-                                about TEXT
+                    id INTEGER PRIMARY KEY,
-                                );""",
+                    osxphotos TEXT,
-            "sql_files_table": """ CREATE TABLE IF NOT EXISTS files (
+                    exportdb TEXT 
-                              id INTEGER PRIMARY KEY,
+                    ); """,
-                              filepath TEXT NOT NULL,
+            """ CREATE TABLE IF NOT EXISTS about (
-                              filepath_normalized TEXT NOT NULL,
+                    id INTEGER PRIMARY KEY,
-                              uuid TEXT,
+                    about TEXT
-                              orig_mode INTEGER,
+                    );""",
-                              orig_size INTEGER,
+            """ CREATE TABLE IF NOT EXISTS files (
-                              orig_mtime REAL,
+                    id INTEGER PRIMARY KEY,
-                              exif_mode INTEGER,
+                    filepath TEXT NOT NULL,
-                              exif_size INTEGER,
+                    filepath_normalized TEXT NOT NULL,
-                              exif_mtime REAL
+                    uuid TEXT,
-                              ); """,
+                    orig_mode INTEGER,
-            "sql_files_table_migrate": """ CREATE TABLE IF NOT EXISTS files_migrate (
+                    orig_size INTEGER,
-                              id INTEGER PRIMARY KEY,
+                    orig_mtime REAL,
-                              filepath TEXT NOT NULL,
+                    exif_mode INTEGER,
-                              filepath_normalized TEXT NOT NULL,
+                    exif_size INTEGER,
-                              uuid TEXT,
+                    exif_mtime REAL
-                              orig_mode INTEGER,
+                    ); """,
-                              orig_size INTEGER,
+            """ CREATE TABLE IF NOT EXISTS runs (
-                              orig_mtime REAL,
+                    id INTEGER PRIMARY KEY,
-                              exif_mode INTEGER,
+                    datetime TEXT,
-                              exif_size INTEGER,
+                    python_path TEXT,
-                              exif_mtime REAL,
+                    script_name TEXT,
-                              UNIQUE(filepath_normalized)
+                    args TEXT,
-                              ); """,
+                    cwd TEXT 
-            "sql_files_migrate": """ INSERT INTO files_migrate SELECT * FROM files;""",
+                    ); """,
-            "sql_files_drop_tables": """ DROP TABLE files;""",
+            """ CREATE TABLE IF NOT EXISTS info (
-            "sql_files_alter": """ ALTER TABLE files_migrate RENAME TO files;""",
+                    id INTEGER PRIMARY KEY,
-            "sql_runs_table": """ CREATE TABLE IF NOT EXISTS runs (
+                    uuid text NOT NULL,
-                             id INTEGER PRIMARY KEY,
+                    json_info JSON 
-                             datetime TEXT,
+                    ); """,
-                             python_path TEXT,
+            """ CREATE TABLE IF NOT EXISTS exifdata (
-                             script_name TEXT,
+                    id INTEGER PRIMARY KEY,
-                             args TEXT,
+                    filepath_normalized TEXT NOT NULL,
-                             cwd TEXT 
+                    json_exifdata JSON 
-                             ); """,
+                    ); """,
-            "sql_info_table": """ CREATE TABLE IF NOT EXISTS info (
+            """ CREATE TABLE IF NOT EXISTS edited (
-                             id INTEGER PRIMARY KEY,
+                    id INTEGER PRIMARY KEY,
-                             uuid text NOT NULL,
+                    filepath_normalized TEXT NOT NULL,
-                             json_info JSON 
+                    mode INTEGER,
-                             ); """,
+                    size INTEGER,
-            "sql_exifdata_table": """ CREATE TABLE IF NOT EXISTS exifdata (
+                    mtime REAL
-                             id INTEGER PRIMARY KEY,
+                    ); """,
-                             filepath_normalized TEXT NOT NULL,
+            """ CREATE TABLE IF NOT EXISTS converted (
-                             json_exifdata JSON 
+                    id INTEGER PRIMARY KEY,
-                             ); """,
+                    filepath_normalized TEXT NOT NULL,
-            "sql_edited_table": """ CREATE TABLE IF NOT EXISTS edited (
+                    mode INTEGER,
-                              id INTEGER PRIMARY KEY,
+                    size INTEGER,
-                              filepath_normalized TEXT NOT NULL,
+                    mtime REAL
-                              mode INTEGER,
+                    ); """,
-                              size INTEGER,
+            """ CREATE TABLE IF NOT EXISTS sidecar (
-                              mtime REAL
+                    id INTEGER PRIMARY KEY,
-                              ); """,
+                    filepath_normalized TEXT NOT NULL,
-            "sql_converted_table": """ CREATE TABLE IF NOT EXISTS converted (
+                    sidecar_data TEXT,
-                              id INTEGER PRIMARY KEY,
+                    mode INTEGER,
-                              filepath_normalized TEXT NOT NULL,
+                    size INTEGER,
-                              mode INTEGER,
+                    mtime REAL
-                              size INTEGER,
+                    ); """,
-                              mtime REAL
+            """ CREATE TABLE IF NOT EXISTS detected_text (
-                              ); """,
+                    id INTEGER PRIMARY KEY,
-            "sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar (
+                    uuid TEXT NOT NULL,
-                              id INTEGER PRIMARY KEY,
+                    text_data JSON
-                              filepath_normalized TEXT NOT NULL,
+                    ); """,
-                              sidecar_data TEXT,
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
-                              mode INTEGER,
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
-                              size INTEGER,
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
-                              mtime REAL
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
-                              ); """,
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
-            "sql_detected_text_table": """ CREATE TABLE IF NOT EXISTS detected_text (
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
-                              id INTEGER PRIMARY KEY,
+            """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
-                              uuid TEXT NOT NULL,
+        ]
-                              text_data JSON
+        # create the tables if needed
-                              ); """,
-            "sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
-            "sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
-            "sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
-            "sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
-            "sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
-            "sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
-            "sql_detected_text_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
-        }
         try:
             c = conn.cursor()
-            for cmd in sql_commands.values():
+            for cmd in sql_commands:
                 c.execute(cmd)
             c.execute(
                 "INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);",
@@ -757,6 +823,19 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+        # perform needed migrations
+        if version[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH:
+            self._migrate_normalized_filepath(conn)
+
+        if version[1] < OSXPHOTOS_EXPORTDB_VERSION:
+            try:
+                c = conn.cursor()
+                # add metadata column to files to support --force-update
+                c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;")
+                conn.commit()
+            except Error as e:
+                logging.warning(e)
+
     def __del__(self):
         """ensure the database connection is closed"""
         try:
@@ -782,6 +861,54 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+    def _normalize_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, lower case"""
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _normalize_filepath_relative(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, relative path (to export dir), lower case"""
+        filepath = str(pathlib.Path(filepath).relative_to(self._path))
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _migrate_normalized_filepath(self, conn):
+        """Fix all filepath_normalized columns for unicode normalization"""
+        # Prior to database version 4.3, filepath_normalized was not normalized for unicode
+        c = conn.cursor()
+        migration_sql = [
+            """ CREATE TABLE IF NOT EXISTS files_migrate (
+                    id INTEGER PRIMARY KEY,
+                    filepath TEXT NOT NULL,
+                    filepath_normalized TEXT NOT NULL,
+                    uuid TEXT,
+                    orig_mode INTEGER,
+                    orig_size INTEGER,
+                    orig_mtime REAL,
+                    exif_mode INTEGER,
+                    exif_size INTEGER,
+                    exif_mtime REAL,
+                    UNIQUE(filepath_normalized)
+                    ); """,
+            """ INSERT INTO files_migrate SELECT * FROM files;""",
+            """ DROP TABLE files;""",
+            """ ALTER TABLE files_migrate RENAME TO files;""",
+        ]
+        for sql in migration_sql:
+            c.execute(sql)
+        conn.commit()
+
+        for table in ["converted", "edited", "exifdata", "files", "sidecar"]:
+            old_values = c.execute(
+                f"SELECT filepath_normalized, id FROM {table}"
+            ).fetchall()
+            new_values = [
+                (self._normalize_filepath(filepath_normalized), id_)
+                for filepath_normalized, id_ in old_values
+            ]
+            c.executemany(
+                f"UPDATE {table} SET filepath_normalized=? WHERE id=?", new_values
+            )
+        conn.commit()
+
 
 class ExportDBInMemory(ExportDB):
     """In memory version of ExportDB
@@ -807,7 +934,7 @@ class ExportDBInMemory(ExportDB):
             conn = self._get_db_connection()
             if not conn:
                 raise Exception("Error getting connection to in-memory database")
-            self._create_db_tables(conn)
+            self._create_or_migrate_db_tables(conn)
             self.was_created = True
             self.was_upgraded = ()
         else:
@@ -830,7 +957,7 @@ class ExportDBInMemory(ExportDB):
             self.was_created = False
             _, exportdb_ver = self._get_database_version(conn)
             if exportdb_ver < OSXPHOTOS_EXPORTDB_VERSION:
-                self._create_db_tables(conn)
+                self._create_or_migrate_db_tables(conn)
                 self.was_upgraded = (exportdb_ver, OSXPHOTOS_EXPORTDB_VERSION)
             else:
                 self.was_upgraded = ()

osxphotos/photoexporter.py

@@ -3,7 +3,6 @@
 
 
 import dataclasses
-import glob
 import hashlib
 import json
 import logging
@@ -33,7 +32,7 @@ from ._constants import (
 )
 from ._version import __version__
 from .datetime_utils import datetime_tz_to_utc
-from .exiftool import ExifTool
+from .exiftool import ExifTool, exiftool_can_write
 from .export_db import ExportDB_ABC, ExportDBNoOp
 from .fileutil import FileUtil
 from .photokit import (
@@ -84,6 +83,7 @@ class ExportOptions:
         export_as_hardlink: (bool, default=False): if True, will hardlink files instead of copying them
         export_db: (ExportDB_ABC): instance of a class that conforms to ExportDB_ABC with methods for getting/setting data related to exported files to compare update state
         fileutil: (FileUtilABC): class that conforms to FileUtilABC with various file utilities
+        force_update: (bool, default=False): if True, will export photo if any metadata has changed but export otherwise would not be triggered (e.g. metadata changed but not using exiftool)
         ignore_date_modified (bool): for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
         ignore_signature (bool, default=False): ignore file signature when used with update (look only at filename)
         increment (bool, default=True): if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists
@@ -129,6 +129,7 @@ class ExportOptions:
     export_as_hardlink: bool = False
     export_db: Optional[ExportDB_ABC] = None
     fileutil: Optional[FileUtil] = None
+    force_update: bool = False
     ignore_date_modified: bool = False
     ignore_signature: bool = False
     increment: bool = True
@@ -450,71 +451,95 @@ class PhotoExporter:
                 dest,
                 options=options,
             )
+        else:
+            verbose(
+                f"Skipping missing {'edited' if options.edited else 'original'} photo {self.photo.original_filename} ({self.photo.uuid})"
+            )
+            all_results.missing.append(dest)
 
         # copy live photo associated .mov if requested
-        if (
+        if export_original and options.live_photo and self.photo.live_photo:
-            export_original
-            and options.live_photo
-            and self.photo.live_photo
-            and staged_files.original_live
-        ):
             live_name = dest.parent / f"{dest.stem}.mov"
-            src_live = staged_files.original_live
+            if staged_files.original_live:
-            all_results += self._export_photo(
+                src_live = staged_files.original_live
-                src_live,
+                all_results += self._export_photo(
-                live_name,
+                    src_live,
-                # don't try to convert the live photo
+                    live_name,
-                options=dataclasses.replace(options, convert_to_jpeg=False),
+                    # don't try to convert the live photo
-            )
+                    options=dataclasses.replace(options, convert_to_jpeg=False),
+                )
+            else:
+                verbose(
+                    f"Skipping missing live photo for {self.photo.original_filename} ({self.photo.uuid})"
+                )
+                all_results.missing.append(live_name)
 
-        if (
+        if export_edited and options.live_photo and self.photo.live_photo:
-            export_edited
-            and options.live_photo
-            and self.photo.live_photo
-            and staged_files.edited_live
-        ):
             live_name = dest.parent / f"{dest.stem}.mov"
-            src_live = staged_files.edited_live
+            if staged_files.edited_live:
-            all_results += self._export_photo(
+                src_live = staged_files.edited_live
-                src_live,
+                all_results += self._export_photo(
-                live_name,
+                    src_live,
-                # don't try to convert the live photo
+                    live_name,
-                options=dataclasses.replace(options, convert_to_jpeg=False),
+                    # don't try to convert the live photo
-            )
+                    options=dataclasses.replace(options, convert_to_jpeg=False),
+                )
+            else:
+                verbose(
+                    f"Skipping missing edited live photo for {self.photo.original_filename} ({self.photo.uuid})"
+                )
+                all_results.missing.append(live_name)
 
         # copy associated RAW image if requested
-        if options.raw_photo and self.photo.has_raw and staged_files.raw:
+        if options.raw_photo and self.photo.has_raw:
-            raw_path = pathlib.Path(staged_files.raw)
+            if staged_files.raw:
-            raw_ext = raw_path.suffix
+                raw_path = pathlib.Path(staged_files.raw)
-            raw_name = dest.parent / f"{dest.stem}{raw_ext}"
+                raw_ext = raw_path.suffix
-            all_results += self._export_photo(
+                raw_name = dest.parent / f"{dest.stem}{raw_ext}"
-                raw_path,
+                all_results += self._export_photo(
-                raw_name,
+                    raw_path,
-                options=options,
+                    raw_name,
-            )
+                    options=options,
+                )
+            else:
+                # guess at most likely raw name
+                raw_ext = get_preferred_uti_extension(self.photo.uti_raw) or "raw"
+                raw_name = dest.parent / f"{dest.stem}.{raw_ext}"
+                all_results.missing.append(raw_name)
+                verbose(
+                    f"Skipping missing raw photo for {self.photo.original_filename} ({self.photo.uuid})"
+                )
 
         # copy preview image if requested
-        if options.preview and staged_files.preview:
+        if options.preview:
-            # Photos keeps multiple different derivatives and path_derivatives returns list of them
+            if staged_files.preview:
-            # first derivative is the largest so export that one
+                # Photos keeps multiple different derivatives and path_derivatives returns list of them
-            preview_path = pathlib.Path(staged_files.preview)
+                # first derivative is the largest so export that one
-            preview_ext = preview_path.suffix
+                preview_path = pathlib.Path(staged_files.preview)
-            preview_name = (
+                preview_ext = preview_path.suffix
-                dest.parent / f"{dest.stem}{options.preview_suffix}{preview_ext}"
+                preview_name = (
-            )
+                    dest.parent / f"{dest.stem}{options.preview_suffix}{preview_ext}"
-            # if original is missing, the filename won't have been incremented so
+                )
-            # need to check here to make sure there aren't duplicate preview files in
+                # if original is missing, the filename won't have been incremented so
-            # the export directory
+                # need to check here to make sure there aren't duplicate preview files in
-            preview_name = (
+                # the export directory
-                preview_name
+                preview_name = (
-                if options.overwrite or options.update
+                    preview_name
-                else pathlib.Path(increment_filename(preview_name))
+                    if any([options.overwrite, options.update, options.force_update])
-            )
+                    else pathlib.Path(increment_filename(preview_name))
-            all_results += self._export_photo(
+                )
-                preview_path,
+                all_results += self._export_photo(
-                preview_name,
+                    preview_path,
-                options=options,
+                    preview_name,
-            )
+                    options=options,
+                )
+            else:
+                # don't know what actual preview suffix would be but most likely jpeg
+                preview_name = dest.parent / f"{dest.stem}{options.preview_suffix}.jpeg"
+                all_results.missing.append(preview_name)
+                verbose(
+                    f"Skipping missing preview photo for {self.photo.original_filename} ({self.photo.uuid})"
+                )
 
         all_results += self._write_sidecar_files(dest=dest, options=options)
 
@@ -566,7 +591,7 @@ class PhotoExporter:
 
         # if overwrite==False and #increment==False, export should fail if file exists
         if dest.exists() and not any(
-            [options.increment, options.update, options.overwrite]
+            [options.increment, options.update, options.force_update, options.overwrite]
         ):
             raise FileExistsError(
                 f"destination exists ({dest}); overwrite={options.overwrite}, increment={options.increment}"
@@ -578,11 +603,13 @@ class PhotoExporter:
         # e.g. exporting sidecar for file1.png and file1.jpeg
         # if file1.png exists and exporting file1.jpeg,
         # dest will be file1 (1).jpeg even though file1.jpeg doesn't exist to prevent sidecar collision
-        if options.increment and not options.update and not options.overwrite:
+        if options.increment and not any(
+            [options.update, options.force_update, options.overwrite]
+        ):
             return pathlib.Path(increment_filename(dest))
 
         # if update and file exists, need to check to see if it's the write file by checking export db
-        if options.update and dest.exists() and src:
+        if (options.update or options.force_update) and dest.exists() and src:
             export_db = options.export_db
             fileutil = options.fileutil
             # destination exists, check to see if destination is the right UUID
@@ -712,7 +739,7 @@ class PhotoExporter:
         # export live_photo .mov file?
         live_photo = bool(options.live_photo and self.photo.live_photo)
 
-        overwrite = options.overwrite or options.update
+        overwrite = any([options.overwrite, options.update, options.force_update])
 
         # figure out which photo version to request
         if options.edited or self.photo.shared:
@@ -820,7 +847,7 @@ class PhotoExporter:
 
         # export live_photo .mov file?
         live_photo = bool(options.live_photo and self.photo.live_photo)
-        overwrite = options.overwrite or options.update
+        overwrite = any([options.overwrite, options.update, options.force_update])
         edited_version = options.edited or self.photo.shared
         # shared photos (in shared albums) show up as not having adjustments (not edited)
         # but Photos is unable to export the "original" as only a jpeg copy is shared in iCloud
@@ -972,18 +999,16 @@ class PhotoExporter:
         fileutil = options.fileutil
         export_db = options.export_db
 
-        if options.update:  # updating
+        if options.update or options.force_update:  # updating
             cmp_touch, cmp_orig = False, False
             if dest_exists:
                 # update, destination exists, but we might not need to replace it...
-                if options.ignore_signature:
+                if options.exiftool:
-                    cmp_orig = True
-                    cmp_touch = fileutil.cmp(
-                        src, dest, mtime1=int(self.photo.date.timestamp())
-                    )
-                elif options.exiftool:
                     sig_exif = export_db.get_stat_exif_for_file(dest_str)
                     cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif)
+                    if cmp_orig:
+                        # if signatures match also need to compare exifdata to see if metadata changed
+                        cmp_orig = not self._should_run_exiftool(dest_str, options)
                     sig_exif = (
                         sig_exif[0],
                         sig_exif[1],
@@ -1000,10 +1025,17 @@ class PhotoExporter:
                     )
                     cmp_touch = fileutil.cmp_file_sig(dest_str, sig_converted)
                 else:
-                    cmp_orig = fileutil.cmp(src, dest)
+                    cmp_orig = options.ignore_signature or fileutil.cmp(src, dest)
                     cmp_touch = fileutil.cmp(
                         src, dest, mtime1=int(self.photo.date.timestamp())
                     )
+                    if options.force_update:
+                        # need to also check the photo's metadata to that in the database
+                        # and if anything changed, we need to update the file
+                        # ony the hex digest of the metadata is stored in the database
+                        cmp_orig = hexdigest(
+                            self.photo.json()
+                        ) == export_db.get_metadata_for_file(dest_str)
 
                 sig_cmp = cmp_touch if options.touch_file else cmp_orig
 
@@ -1017,7 +1049,7 @@ class PhotoExporter:
                         if sig_edited != (None, None, None)
                         else False
                     )
-                    sig_cmp = sig_cmp and cmp_edited
+                    sig_cmp = sig_cmp and (options.force_update or cmp_edited)
 
                 if (options.export_as_hardlink and dest.samefile(src)) or (
                     not options.export_as_hardlink
@@ -1060,7 +1092,9 @@ class PhotoExporter:
             edited_stat = (
                 fileutil.file_sig(src) if options.edited else (None, None, None)
             )
-            if dest_exists and (options.update or options.overwrite):
+            if dest_exists and any(
+                [options.overwrite, options.update, options.force_update]
+            ):
                 # need to remove the destination first
                 try:
                     fileutil.unlink(dest)
@@ -1103,13 +1137,15 @@ class PhotoExporter:
                         f"Error copying file {src} to {dest_str}: {e} ({lineno(__file__)})"
                     ) from e
 
+        json_info = self.photo.json()
         export_db.set_data(
             filename=dest_str,
             uuid=self.photo.uuid,
             orig_stat=fileutil.file_sig(dest_str),
             converted_stat=converted_stat,
             edited_stat=edited_stat,
-            info_json=self.photo.json(),
+            info_json=json_info,
+            metadata=hexdigest(json_info),
         )
 
         return ExportResults(
@@ -1209,10 +1245,13 @@ class PhotoExporter:
                 sidecar_filename
             )
             write_sidecar = (
-                not options.update
+                not (options.update or options.force_update)
-                or (options.update and not sidecar_filename.exists())
                 or (
-                    options.update
+                    (options.update or options.force_update)
+                    and not sidecar_filename.exists()
+                )
+                or (
+                    (options.update or options.force_update)
                     and (sidecar_digest != old_sidecar_digest)
                     or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig)
                 )
@@ -1260,31 +1299,27 @@ class PhotoExporter:
 
         exiftool_results = ExportResults()
 
+        # don't try to write if unsupported file type for exiftool
+        if not exiftool_can_write(os.path.splitext(src)[-1]):
+            exiftool_results.exiftool_warning.append(
+                (
+                    dest,
+                    f"Unsupported file type for exiftool, skipping exiftool for {dest}",
+                )
+            )
+            # set file signature so the file doesn't get re-exported with --update
+            export_db.set_data(
+                dest,
+                uuid=self.photo.uuid,
+                exif_stat=fileutil.file_sig(src),
+                exif_json=self._exiftool_json_sidecar(options=options),
+            )
+            return exiftool_results
+
         # determine if we need to write the exif metadata
         # if we are not updating, we always write
         # else, need to check the database to determine if we need to write
-        run_exiftool = not options.update
+        run_exiftool = self._should_run_exiftool(dest, options)
-        current_data = "foo"
-        if options.update:
-            files_are_different = False
-            old_data = export_db.get_exifdata_for_file(dest)
-            if old_data is not None:
-                old_data = json.loads(old_data)[0]
-                current_data = json.loads(self._exiftool_json_sidecar(options=options))[
-                    0
-                ]
-                if old_data != current_data:
-                    files_are_different = True
-
-            if old_data is None or files_are_different:
-                # didn't have old data, assume we need to write it
-                # or files were different
-                run_exiftool = True
-            else:
-                verbose(
-                    f"Skipped up to date exiftool metadata for {pathlib.Path(dest).name}"
-                )
-
         if run_exiftool:
             verbose(f"Writing metadata with exiftool for {pathlib.Path(dest).name}")
             if not options.dry_run:
@@ -1303,8 +1338,32 @@ class PhotoExporter:
             )
             exiftool_results.exif_updated.append(dest)
             exiftool_results.to_touch.append(dest)
+        else:
+            verbose(
+                f"Skipped up to date exiftool metadata for {pathlib.Path(dest).name}"
+            )
         return exiftool_results
 
+    def _should_run_exiftool(self, dest, options: ExportOptions) -> bool:
+        """Return True if exiftool should be run to update metadata"""
+        run_exiftool = not (options.update or options.force_update)
+        if options.update or options.force_update:
+            files_are_different = False
+            old_data = options.export_db.get_exifdata_for_file(dest)
+            if old_data is not None:
+                old_data = json.loads(old_data)[0]
+                current_data = json.loads(self._exiftool_json_sidecar(options=options))[
+                    0
+                ]
+                if old_data != current_data:
+                    files_are_different = True
+
+            if old_data is None or files_are_different:
+                # didn't have old data, assume we need to write it
+                # or files were different
+                run_exiftool = True
+        return run_exiftool
+
     def _write_exif_data(self, filepath: str, options: ExportOptions):
         """write exif data to image file at filepath
 

osxphotos/photoinfo.py

@@ -1728,7 +1728,8 @@ class PhotoInfo:
             if isinstance(o, (datetime.date, datetime.datetime)):
                 return o.isoformat()
 
-        return json.dumps(self.asdict(), sort_keys=True, default=default)
+        dict_data = self.asdict()
+        return json.dumps(dict_data, sort_keys=True, default=default)
 
     def __eq__(self, other):
         """Compare two PhotoInfo objects for equality"""

osxphotos/photosdb/photosdb.py

@@ -39,6 +39,7 @@ from .._constants import (
     _PHOTOS_5_PROJECT_ALBUM_KIND,
     _PHOTOS_5_ROOT_FOLDER_KIND,
     _PHOTOS_5_SHARED_ALBUM_KIND,
+    _PHOTOS_5_VERSION,
     _TESTED_OS_VERSIONS,
     _UNKNOWN_PERSON,
     BURST_KEY,
@@ -659,14 +660,18 @@ class PhotosDB:
 
         for person in c:
             pk = person[0]
-            fullname = person[2] if person[2] is not None else _UNKNOWN_PERSON
+            fullname = (
+                normalize_unicode(person[2])
+                if person[2] is not None
+                else _UNKNOWN_PERSON
+            )
             self._dbpersons_pk[pk] = {
                 "pk": pk,
                 "uuid": person[1],
                 "fullname": fullname,
                 "facecount": person[3],
                 "keyface": person[5],
-                "displayname": person[4],
+                "displayname": normalize_unicode(person[4]),
                 "photo_uuid": None,
                 "keyface_uuid": None,
             }
@@ -733,13 +738,6 @@ class PhotosDB:
             except KeyError:
                 self._dbfaces_pk[pk] = [uuid]
 
-        if _debug():
-            logging.debug(f"Finished walking through persons")
-            logging.debug(pformat(self._dbpersons_pk))
-            logging.debug(pformat(self._dbpersons_fullname))
-            logging.debug(pformat(self._dbfaces_pk))
-            logging.debug(pformat(self._dbfaces_uuid))
-
         # Get info on albums
         verbose("Processing albums.")
         c.execute(
@@ -876,14 +874,6 @@ class PhotosDB:
             else:
                 self._dbalbum_folders[album] = {}
 
-        if _debug():
-            logging.debug(f"Finished walking through albums")
-            logging.debug(pformat(self._dbalbums_album))
-            logging.debug(pformat(self._dbalbums_uuid))
-            logging.debug(pformat(self._dbalbum_details))
-            logging.debug(pformat(self._dbalbum_folders))
-            logging.debug(pformat(self._dbfolder_details))
-
         # Get info on keywords
         verbose("Processing keywords.")
         c.execute(
@@ -899,13 +889,16 @@ class PhotosDB:
                 RKMaster.uuid = RKVersion.masterUuid
             """
         )
-        for keyword in c:
+        for keyword_title, keyword_uuid, _ in c:
-            if not keyword[1] in self._dbkeywords_uuid:
+            keyword_title = normalize_unicode(keyword_title)
-                self._dbkeywords_uuid[keyword[1]] = []
+            try:
-            if not keyword[0] in self._dbkeywords_keyword:
+                self._dbkeywords_uuid[keyword_uuid].append(keyword_title)
-                self._dbkeywords_keyword[keyword[0]] = []
+            except KeyError:
-            self._dbkeywords_uuid[keyword[1]].append(keyword[0])
+                self._dbkeywords_uuid[keyword_uuid] = [keyword_title]
-            self._dbkeywords_keyword[keyword[0]].append(keyword[1])
+            try:
+                self._dbkeywords_keyword[keyword_title].append(keyword_uuid)
+            except KeyError:
+                self._dbkeywords_keyword[keyword_title] = [keyword_uuid]
 
         # Get info on disk volumes
         c.execute("select RKVolume.modelId, RKVolume.name from RKVolume")
@@ -1027,13 +1020,11 @@ class PhotosDB:
 
         for row in c:
             uuid = row[0]
-            if _debug():
-                logging.debug(f"uuid = '{uuid}, master = '{row[2]}")
             self._dbphotos[uuid] = {}
             self._dbphotos[uuid]["_uuid"] = uuid  # stored here for easier debugging
             self._dbphotos[uuid]["modelID"] = row[1]
             self._dbphotos[uuid]["masterUuid"] = row[2]
-            self._dbphotos[uuid]["filename"] = row[3]
+            self._dbphotos[uuid]["filename"] = normalize_unicode(row[3])
 
             # There are sometimes negative values for lastmodifieddate in the database
             # I don't know what these mean but they will raise exception in datetime if
@@ -1272,13 +1263,13 @@ class PhotosDB:
             info["volumeId"] = row[1]
             info["imagePath"] = row[2]
             info["isMissing"] = row[3]
-            info["originalFilename"] = row[4]
+            info["originalFilename"] = normalize_unicode(row[4])
             info["UTI"] = row[5]
             info["modelID"] = row[6]
             info["fileSize"] = row[7]
             info["isTrulyRAW"] = row[8]
             info["alternateMasterUuid"] = row[9]
-            info["filename"] = row[10]
+            info["filename"] = normalize_unicode(row[10])
             self._dbphotos_master[uuid] = info
 
         # get details needed to find path of the edited photos
@@ -1550,39 +1541,6 @@ class PhotosDB:
 
         # done processing, dump debug data if requested
         verbose("Done processing details from Photos library.")
-        if _debug():
-            logging.debug("Faces (_dbfaces_uuid):")
-            logging.debug(pformat(self._dbfaces_uuid))
-
-            logging.debug("Persons (_dbpersons_pk):")
-            logging.debug(pformat(self._dbpersons_pk))
-
-            logging.debug("Keywords by uuid (_dbkeywords_uuid):")
-            logging.debug(pformat(self._dbkeywords_uuid))
-
-            logging.debug("Keywords by keyword (_dbkeywords_keywords):")
-            logging.debug(pformat(self._dbkeywords_keyword))
-
-            logging.debug("Albums by uuid (_dbalbums_uuid):")
-            logging.debug(pformat(self._dbalbums_uuid))
-
-            logging.debug("Albums by album (_dbalbums_albums):")
-            logging.debug(pformat(self._dbalbums_album))
-
-            logging.debug("Album details (_dbalbum_details):")
-            logging.debug(pformat(self._dbalbum_details))
-
-            logging.debug("Album titles (_dbalbum_titles):")
-            logging.debug(pformat(self._dbalbum_titles))
-
-            logging.debug("Volumes (_dbvolumes):")
-            logging.debug(pformat(self._dbvolumes))
-
-            logging.debug("Photos (_dbphotos):")
-            logging.debug(pformat(self._dbphotos))
-
-            logging.debug("Burst Photos (dbphotos_burst:")
-            logging.debug(pformat(self._dbphotos_burst))
 
     def _build_album_folder_hierarchy_4(self, uuid, folders=None):
         """recursively build folder/album hierarchy
@@ -1673,7 +1631,7 @@ class PhotosDB:
         for person in c:
             pk = person[0]
             fullname = (
-                person[2]
+                normalize_unicode(person[2])
                 if (person[2] != "" and person[2] is not None)
                 else _UNKNOWN_PERSON
             )
@@ -1683,7 +1641,7 @@ class PhotosDB:
                 "fullname": fullname,
                 "facecount": person[3],
                 "keyface": person[4],
-                "displayname": person[5],
+                "displayname": normalize_unicode(person[5]),
                 "photo_uuid": None,
                 "keyface_uuid": None,
             }
@@ -1747,13 +1705,6 @@ class PhotosDB:
             except KeyError:
                 self._dbfaces_pk[pk] = [uuid]
 
-        if _debug():
-            logging.debug(f"Finished walking through persons")
-            logging.debug(pformat(self._dbpersons_pk))
-            logging.debug(pformat(self._dbpersons_fullname))
-            logging.debug(pformat(self._dbfaces_pk))
-            logging.debug(pformat(self._dbfaces_uuid))
-
         # get details about albums
         verbose("Processing albums.")
         c.execute(
@@ -1870,13 +1821,6 @@ class PhotosDB:
                 # shared albums can't be in folders
                 self._dbalbum_folders[album] = []
 
-        if _debug():
-            logging.debug(f"Finished walking through albums")
-            logging.debug(pformat(self._dbalbums_album))
-            logging.debug(pformat(self._dbalbums_uuid))
-            logging.debug(pformat(self._dbalbum_details))
-            logging.debug(pformat(self._dbalbum_folders))
-
         # get details on keywords
         verbose("Processing keywords.")
         c.execute(
@@ -1886,29 +1830,22 @@ class PhotosDB:
                 JOIN Z_1KEYWORDS ON Z_1KEYWORDS.Z_1ASSETATTRIBUTES = ZADDITIONALASSETATTRIBUTES.Z_PK 
                 JOIN ZKEYWORD ON ZKEYWORD.Z_PK = {keyword_join} """
         )
-        for keyword in c:
+        for keyword_title, keyword_uuid in c:
-            keyword_title = normalize_unicode(keyword[0])
+            keyword_title = normalize_unicode(keyword_title)
-            if not keyword[1] in self._dbkeywords_uuid:
+            try:
-                self._dbkeywords_uuid[keyword[1]] = []
+                self._dbkeywords_uuid[keyword_uuid].append(keyword_title)
-            if not keyword_title in self._dbkeywords_keyword:
+            except KeyError:
-                self._dbkeywords_keyword[keyword_title] = []
+                self._dbkeywords_uuid[keyword_uuid] = [keyword_title]
-            self._dbkeywords_uuid[keyword[1]].append(keyword[0])
+            try:
-            self._dbkeywords_keyword[keyword_title].append(keyword[1])
+                self._dbkeywords_keyword[keyword_title].append(keyword_uuid)
-
+            except KeyError:
-        if _debug():
+                self._dbkeywords_keyword[keyword_title] = [keyword_uuid]
-            logging.debug(f"Finished walking through keywords")
-            logging.debug(pformat(self._dbkeywords_keyword))
-            logging.debug(pformat(self._dbkeywords_uuid))
 
         # get details on disk volumes
         c.execute("SELECT ZUUID, ZNAME from ZFILESYSTEMVOLUME")
         for vol in c:
             self._dbvolumes[vol[0]] = vol[1]
 
-        if _debug():
-            logging.debug(f"Finished walking through volumes")
-            logging.debug(self._dbvolumes)
-
         # get details about photos
         verbose("Processing photo details.")
         c.execute(
@@ -2042,8 +1979,8 @@ class PhotosDB:
 
             info["hidden"] = row[9]
             info["favorite"] = row[10]
-            info["originalFilename"] = row[3]
+            info["originalFilename"] = normalize_unicode(row[3])
-            info["filename"] = row[12]
+            info["filename"] = normalize_unicode(row[12])
             info["directory"] = row[11]
 
             # set latitude and longitude
@@ -2521,48 +2458,6 @@ class PhotosDB:
 
         # done processing, dump debug data if requested
         verbose("Done processing details from Photos library.")
-        if _debug():
-            logging.debug("Faces (_dbfaces_uuid):")
-            logging.debug(pformat(self._dbfaces_uuid))
-
-            logging.debug("Persons (_dbpersons_pk):")
-            logging.debug(pformat(self._dbpersons_pk))
-
-            logging.debug("Keywords by uuid (_dbkeywords_uuid):")
-            logging.debug(pformat(self._dbkeywords_uuid))
-
-            logging.debug("Keywords by keyword (_dbkeywords_keywords):")
-            logging.debug(pformat(self._dbkeywords_keyword))
-
-            logging.debug("Albums by uuid (_dbalbums_uuid):")
-            logging.debug(pformat(self._dbalbums_uuid))
-
-            logging.debug("Albums by album (_dbalbums_albums):")
-            logging.debug(pformat(self._dbalbums_album))
-
-            logging.debug("Album details (_dbalbum_details):")
-            logging.debug(pformat(self._dbalbum_details))
-
-            logging.debug("Album titles (_dbalbum_titles):")
-            logging.debug(pformat(self._dbalbum_titles))
-
-            logging.debug("Album folders (_dbalbum_folders):")
-            logging.debug(pformat(self._dbalbum_folders))
-
-            logging.debug("Album parent folders (_dbalbum_parent_folders):")
-            logging.debug(pformat(self._dbalbum_parent_folders))
-
-            logging.debug("Albums pk (_dbalbums_pk):")
-            logging.debug(pformat(self._dbalbums_pk))
-
-            logging.debug("Volumes (_dbvolumes):")
-            logging.debug(pformat(self._dbvolumes))
-
-            logging.debug("Photos (_dbphotos):")
-            logging.debug(pformat(self._dbphotos))
-
-            logging.debug("Burst Photos (dbphotos_burst:")
-            logging.debug(pformat(self._dbphotos_burst))
 
     def _process_moments(self):
         """Process data from ZMOMENT table"""
@@ -2623,8 +2518,8 @@ class PhotosDB:
             moment_info["modificationDate"] = row[6]
             moment_info["representativeDate"] = row[7]
             moment_info["startDate"] = row[8]
-            moment_info["subtitle"] = row[9]
+            moment_info["subtitle"] = normalize_unicode(row[9])
-            moment_info["title"] = row[10]
+            moment_info["title"] = normalize_unicode(row[10])
             moment_info["uuid"] = row[11]
 
             # if both lat/lon == -180, then it means location undefined
@@ -3027,6 +2922,7 @@ class PhotosDB:
             if keywords:
                 keyword_set = set()
                 for keyword in keywords:
+                    keyword = normalize_unicode(keyword)
                     if keyword in self._dbkeywords_keyword:
                         keyword_set.update(self._dbkeywords_keyword[keyword])
                 photos_sets.append(keyword_set)
@@ -3034,6 +2930,7 @@ class PhotosDB:
             if persons:
                 person_set = set()
                 for person in persons:
+                    person = normalize_unicode(person)
                     if person in self._dbpersons_fullname:
                         for pk in self._dbpersons_fullname[person]:
                             try:
@@ -3076,8 +2973,6 @@ class PhotosDB:
                 ):
                     info = PhotoInfo(db=self, uuid=p, info=self._dbphotos[p])
                     photoinfo.append(info)
-        if _debug:
-            logging.debug(f"photoinfo: {pformat(photoinfo)}")
 
         return photoinfo
 
@@ -3414,23 +3309,35 @@ class PhotosDB:
                 # case-insensitive
                 for n in name:
                     n = n.lower()
-                    photo_list.extend(
+                    if self._db_version >= _PHOTOS_5_VERSION:
-                        [
+                        # search only original_filename (#594)
-                            p
+                        photo_list.extend(
-                            for p in photos
+                            [p for p in photos if n in p.original_filename.lower()]
-                            if n in p.filename.lower()
+                        )
-                            or n in p.original_filename.lower()
+                    else:
-                        ]
+                        photo_list.extend(
-                    )
+                            [
+                                p
+                                for p in photos
+                                if n in p.filename.lower()
+                                or n in p.original_filename.lower()
+                            ]
+                        )
             else:
                 for n in name:
-                    photo_list.extend(
+                    if self._db_version >= _PHOTOS_5_VERSION:
-                        [
+                        # search only original_filename (#594)
-                            p
+                        photo_list.extend(
-                            for p in photos
+                            [p for p in photos if n in p.original_filename]
-                            if n in p.filename or n in p.original_filename
+                        )
-                        ]
+                    else:
-                    )
+                        photo_list.extend(
+                            [
+                                p
+                                for p in photos
+                                if n in p.filename or n in p.original_filename
+                            ]
+                        )
             photos = photo_list
 
         if options.min_size:

tests/Test-10.15.7.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>585B80BF-8D1F-55EF-A9E8-6CF4E5523959</string>
 	<key>pid</key>
-	<integer>1961</integer>
+	<integer>14817</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2021-09-14T04:40:41Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2021-07-20T05:48:08Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2021-07-20T05:47:59Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2021-09-14T04:40:43Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 </dict>
 </plist>

tests/conftest.py

@@ -40,7 +40,7 @@ else:
 
 @pytest.fixture(autouse=True)
 def reset_singletons():
-    """ Need to clean up any ExifTool singletons between tests """
+    """Need to clean up any ExifTool singletons between tests"""
     _ExifToolProc.instance = None
 
 
@@ -73,7 +73,7 @@ def pytest_collection_modifyitems(config, items):
 
 
 def copy_photos_library(photos_library=TEST_LIBRARY, delay=0):
-    """ copy the test library and open Photos, returns path to copied library """
+    """copy the test library and open Photos, returns path to copied library"""
     script = AppleScript(
         """
         tell application "Photos"
@@ -118,3 +118,9 @@ def copy_photos_library(photos_library=TEST_LIBRARY, delay=0):
 @pytest.fixture
 def addalbum_library():
     copy_photos_library(delay=10)
+
+
+def copy_photos_library_to_path(photos_library_path: str, dest_path: str) -> str:
+    """Copy a photos library to a folder"""
+    ditto(photos_library_path, dest_path)
+    return dest_path

tests/test_albums_folders_catalina_10_15_7.py

@@ -21,6 +21,7 @@ FOLDER_ALBUM_DICT = {
 ALBUM_NAMES = [
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum",
     "2019-10/11 Paris Clermont",
+    "Água",
     "AlbumInFolder",
     "EmptyAlbum",
     "I have a deleted twin",
@@ -38,6 +39,7 @@ ALBUM_NAMES = [
 ALBUM_PARENT_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": None,
     "2019-10/11 Paris Clermont": None,
+    "Água": None,
     "AlbumInFolder": "SubFolder2",
     "EmptyAlbum": None,
     "I have a deleted twin": None,
@@ -54,6 +56,7 @@ ALBUM_PARENT_DICT = {
 ALBUM_FOLDER_NAMES_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": [],
     "2019-10/11 Paris Clermont": [],
+    "Água": [],
     "AlbumInFolder": ["Folder1", "SubFolder2"],
     "EmptyAlbum": [],
     "I have a deleted twin": [],
@@ -70,6 +73,7 @@ ALBUM_FOLDER_NAMES_DICT = {
 ALBUM_LEN_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": 1,
     "2019-10/11 Paris Clermont": 1,
+    "Água": 3,
     "AlbumInFolder": 2,
     "EmptyAlbum": 0,
     "I have a deleted twin": 1,
@@ -103,6 +107,11 @@ ALBUM_PHOTO_UUID_DICT = {
         "4D521201-92AC-43E5-8F7C-59BC41C37A96",
         "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
     ],
+    "Água": [
+        "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091",
+        "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
+        "54E76FCB-D353-4557-9997-0A457BCB4D48",
+    ],
 }
 
 UUID_DICT = {

tests/test_catalina_10_15_7.py

@@ -24,10 +24,10 @@ PHOTOS_DB = "tests/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_DB_PATH = "/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_LIBRARY_PATH = "/Test-10.15.7.photoslibrary"
 
-PHOTOS_DB_LEN = 25
+PHOTOS_DB_LEN = 29
-PHOTOS_NOT_IN_TRASH_LEN = 23
+PHOTOS_NOT_IN_TRASH_LEN = 27
 PHOTOS_IN_TRASH_LEN = 2
-PHOTOS_DB_IMPORT_SESSIONS = 17
+PHOTOS_DB_IMPORT_SESSIONS = 21
 
 KEYWORDS = [
     "Kids",
@@ -72,6 +72,7 @@ ALBUMS = [
     "Sorted Oldest First",
     "Sorted Title",
     "Test Album",  # there are 2 albums named "Test Album" for testing duplicate album names
+    "Água",
 ]
 KEYWORDS_DICT = {
     "Drink": 2,
@@ -115,6 +116,7 @@ ALBUM_DICT = {
     "Sorted Oldest First": 3,
     "Sorted Title": 3,
     "Test Album": 2,
+    "Água": 3,
 }  # Note: there are 2 albums named "Test Album" for testing duplicate album names
 
 UUID_DICT = {
@@ -1091,7 +1093,7 @@ def test_from_to_date(photosdb):
     time.tzset()
 
     photos = photosdb.photos(from_date=datetime.datetime(2018, 10, 28))
-    assert len(photos) == 16
+    assert len(photos) == 20
 
     photos = photosdb.photos(to_date=datetime.datetime(2018, 10, 28))
     assert len(photos) == 7

tests/test_export_db.py

@@ -7,6 +7,7 @@ import pytest
 EXIF_DATA = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "EXIF:ImageDescription": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Title": "Elder Park", "EXIF:GPSLatitude": "34 deg 55' 8.01\" S", "EXIF:GPSLongitude": "138 deg 35' 48.70\" E", "Composite:GPSPosition": "34 deg 55' 8.01\" S, 138 deg 35' 48.70\" E", "EXIF:GPSLatitudeRef": "South", "EXIF:GPSLongitudeRef": "East", "EXIF:DateTimeOriginal": "2017:06:20 17:18:56", "EXIF:OffsetTimeOriginal": "+09:30", "EXIF:ModifyDate": "2020:05:18 14:42:04"}]"""
 INFO_DATA = """{"uuid": "3DD2C897-F19E-4CA6-8C22-B027D5A71907", "filename": "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "original_filename": "IMG_4547.jpg", "date": "2017-06-20T17:18:56.518000+09:30", "description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "title": "Elder Park", "keywords": [], "labels": ["Statue", "Art"], "albums": ["AlbumInFolder"], "folders": {"AlbumInFolder": ["Folder1", "SubFolder2"]}, "persons": [], "path": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/originals/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "ismissing": false, "hasadjustments": true, "external_edit": false, "favorite": false, "hidden": false, "latitude": -34.91889167000001, "longitude": 138.59686167, "path_edited": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/resources/renders/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907_1_201_a.jpeg", "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null, "date_modified": "2020-05-18T14:42:04.608664+09:30", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Elder Park, Adelaide, South Australia, Australia, River Torrens", "names": {"field0": [], "country": ["Australia"], "state_province": ["South Australia"], "sub_administrative_area": ["Adelaide"], "city": ["Adelaide", "Adelaide"], "field5": [], "additional_city_info": ["Adelaide CBD", "Tarndanya"], "ocean": [], "area_of_interest": ["Elder Park", ""], "inland_water": ["River Torrens", "River Torrens"], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": ["River Torrens", "River Torrens"]}, "country_code": "AU", "ishome": false, "address_str": "River Torrens, Adelaide SA, Australia", "address": {"street": null, "sub_locality": "Tarndanya", "city": "Adelaide", "sub_administrative_area": "Adelaide", "state_province": "SA", "postal_code": null, "country": "Australia", "iso_country_code": "AU"}}, "exif": {"flash_fired": false, "iso": 320, "metering_mode": 3, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.058823529411764705, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}"""
 SIDECAR_DATA = """FOO_BAR"""
+METADATA_DATA = "FIZZ"
 
 EXIF_DATA2 = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "XMP:Title": "St. James's Park", "XMP:TagsList": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "IPTC:Keywords": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "XMP:Subject": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "EXIF:GPSLatitude": "51 deg 30' 12.86\" N", "EXIF:GPSLongitude": "0 deg 7' 54.50\" W", "Composite:GPSPosition": "51 deg 30' 12.86\" N, 0 deg 7' 54.50\" W", "EXIF:GPSLatitudeRef": "North", "EXIF:GPSLongitudeRef": "West", "EXIF:DateTimeOriginal": "2018:10:13 09:18:12", "EXIF:OffsetTimeOriginal": "-04:00", "EXIF:ModifyDate": "2019:12:08 14:06:44"}]"""
 INFO_DATA2 = """{"uuid": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529", "filename": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "original_filename": "IMG_8440.JPG", "date": "2019-06-11T11:42:06.711805-07:00", "description": null, "title": null, "keywords": [], "labels": ["Sky", "Cloudy", "Fence", "Land", "Outdoor", "Park", "Amusement Park", "Roller Coaster"], "albums": [], "folders": {}, "persons": [], "path": "/Volumes/MacBook Catalina - Data/Users/rhet/Pictures/Photos Library.photoslibrary/originals/F/F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 33.81558666666667, "longitude": -117.99298, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": true, "incloud": true, "date_modified": "2019-10-14T00:51:47.141950-07:00", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Adventure City, Stanton, California, United States", "names": {"field0": [], "country": ["United States"], "state_province": ["California"], "sub_administrative_area": ["Orange"], "city": ["Stanton", "Anaheim", "Anaheim"], "field5": [], "additional_city_info": ["West Anaheim"], "ocean": [], "area_of_interest": ["Adventure City", "Adventure City"], "inland_water": [], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": []}, "country_code": "US", "ishome": false, "address_str": "Adventure City, 1240 S Beach Blvd, Anaheim, CA  92804, United States", "address": {"street": "1240 S Beach Blvd", "sub_locality": "West Anaheim", "city": "Stanton", "sub_administrative_area": "Orange", "state_province": "CA", "postal_code": "92804", "country": "United States", "iso_country_code": "US"}}, "exif": {"flash_fired": false, "iso": 25, "metering_mode": 5, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.0004940711462450593, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}"""
@@ -64,6 +65,7 @@ def test_export_db():
         (10, 11, 12),
         INFO_DATA,
         EXIF_DATA,
+        METADATA_DATA,
     )
     assert db.get_uuid_for_file(filepath2) == "BAR-FOO"
     assert db.get_info_for_uuid("BAR-FOO") == INFO_DATA
@@ -73,6 +75,7 @@ def test_export_db():
     assert db.get_stat_converted_for_file(filepath2) == (7, 8, 9)
     assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12)
     assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"])
+    assert db.get_metadata_for_file(filepath2) == METADATA_DATA
 
     # test set_data value=None doesn't overwrite existing data
     db.set_data(
@@ -84,6 +87,7 @@ def test_export_db():
         None,
         None,
         None,
+        None,
     )
     assert db.get_uuid_for_file(filepath2) == "BAR-FOO"
     assert db.get_info_for_uuid("BAR-FOO") == INFO_DATA
@@ -93,6 +97,7 @@ def test_export_db():
     assert db.get_stat_converted_for_file(filepath2) == (7, 8, 9)
     assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12)
     assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"])
+    assert db.get_metadata_for_file(filepath2) == METADATA_DATA
 
     # close and re-open
     db.close()
@@ -107,6 +112,8 @@ def test_export_db():
     assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12)
     assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"])
     assert json.loads(db.get_detected_text_for_uuid("FOO-BAR")) == [["foo", 0.5]]
+    assert db.get_metadata_for_file(filepath2) == METADATA_DATA
+
 
     # update data
     db.set_uuid_for_file(filepath, "FUBAR")
@@ -148,9 +155,10 @@ def test_export_db_no_op():
     db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15))
     assert db.get_sidecar_for_file(filepath) == (None, (None, None, None))
     assert db.get_previous_uuids() == []
-
     db.set_detected_text_for_uuid("FOO-BAR", json.dumps([["foo", 0.5]]))
     assert db.get_detected_text_for_uuid("FOO-BAR") is None
+    db.set_metadata_for_file(filepath, METADATA_DATA)
+    assert db.get_metadata_for_file(filepath) is None
 
     # test set_data which sets all at the same time
     filepath2 = os.path.join(tempdir.name, "test2.jpg")
@@ -163,6 +171,7 @@ def test_export_db_no_op():
         (10, 11, 12),
         INFO_DATA,
         EXIF_DATA,
+        METADATA_DATA,
     )
     assert db.get_uuid_for_file(filepath2) is None
     assert db.get_info_for_uuid("BAR-FOO") is None
@@ -172,6 +181,7 @@ def test_export_db_no_op():
     assert db.get_stat_converted_for_file(filepath) is None
     assert db.get_stat_edited_for_file(filepath) is None
     assert db.get_previous_uuids() == []
+    assert db.get_metadata_for_file(filepath) is None
 
     # update data
     db.set_uuid_for_file(filepath, "FUBAR")
@@ -207,7 +217,7 @@ def test_export_db_in_memory():
     db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15))
     assert db.get_previous_uuids() == ["FOO-BAR"]
     db.set_detected_text_for_uuid("FOO-BAR", json.dumps([["foo", 0.5]]))
-
+    db.set_metadata_for_file(filepath, METADATA_DATA)
     db.close()
 
     dbram = ExportDBInMemory(dbname, tempdir.name)
@@ -226,6 +236,7 @@ def test_export_db_in_memory():
     assert dbram.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15))
     assert dbram.get_previous_uuids() == ["FOO-BAR"]
     assert json.loads(dbram.get_detected_text_for_uuid("FOO-BAR")) == [["foo", 0.5]]
+    assert dbram.get_metadata_for_file(filepath) == METADATA_DATA
 
     # change a value
     dbram.set_uuid_for_file(filepath, "FUBAR")
@@ -237,6 +248,7 @@ def test_export_db_in_memory():
     dbram.set_stat_edited_for_file(filepath, (4, 5, 6))
     dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22))
     dbram.set_detected_text_for_uuid("FUBAR", json.dumps([["bar", 0.5]]))
+    dbram.set_metadata_for_file(filepath, "FUBAR")
 
     assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR"
     assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2
@@ -248,6 +260,7 @@ def test_export_db_in_memory():
     assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22))
     assert dbram.get_previous_uuids() == ["FUBAR"]
     assert json.loads(dbram.get_detected_text_for_uuid("FUBAR")) == [["bar", 0.5]]
+    assert dbram.get_metadata_for_file(filepath) == "FUBAR"
 
     dbram.close()
 
@@ -265,6 +278,7 @@ def test_export_db_in_memory():
 
     assert db.get_info_for_uuid("FUBAR") is None
     assert db.get_detected_text_for_uuid("FUBAR") is None
+    assert db.get_metadata_for_file(filepath) == METADATA_DATA
 
 
 def test_export_db_in_memory_nofile():
@@ -295,6 +309,7 @@ def test_export_db_in_memory_nofile():
     dbram.set_stat_edited_for_file(filepath, (4, 5, 6))
     dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22))
     dbram.set_detected_text_for_uuid("FUBAR", json.dumps([["bar", 0.5]]))
+    dbram.set_metadata_for_file(filepath, METADATA_DATA)
 
     assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR"
     assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2
@@ -306,5 +321,6 @@ def test_export_db_in_memory_nofile():
     assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22))
     assert dbram.get_previous_uuids() == ["FUBAR"]
     assert json.loads(dbram.get_detected_text_for_uuid("FUBAR")) == [["bar", 0.5]]
+    assert dbram.get_metadata_for_file(filepath) == METADATA_DATA
 
     dbram.close()

utils/exiftool_supported_types.py

@@ -0,0 +1,57 @@
+"""Read the "Supported File Types" table from exiftool.org and build a json file from the table"""
+
+import json
+import sys
+
+import requests
+from bs4 import BeautifulSoup
+
+
+if __name__ == "__main__":
+    url = "https://www.exiftool.org/"
+    json_file = "exiftool_filetypes.json"
+
+    html_content = requests.get(url).text
+
+    soup = BeautifulSoup(html_content, "html.parser")
+
+    # uncomment to see all table classes
+    # print("Classes of each table:")
+    # for table in soup.find_all("table"):
+    #     print(table.get("class"))
+
+    # strip footnotes in <span> tags
+    for span_tag in soup.findAll("span"):
+        span_tag.replace_with("")
+
+    # find the table for Supported File Types
+    table = soup.find("table", class_="sticky tight sm bm")
+
+    # get table headers
+    table_headers = [tx.text.lower() for tx in table.find_all("th")]
+
+    # get table data
+    table_data = []
+    for tr in table.find_all("tr"):
+        if row := [td.text for td in tr.find_all("td")]:
+            table_data.append(row)
+
+    # make a dictionary of the table data
+    supported_filetypes = {}
+    for row in table_data:
+        row_dict = dict(zip(table_headers, row))
+        for key, value in row_dict.items():
+            if value == "-":
+                row_dict[key] = None
+        row_dict["file type"] = row_dict["file type"].split(",")
+        row_dict["file type"] = [ft.strip() for ft in row_dict["file type"]]
+        row_dict["read"] = "R" in row_dict["support"]
+        row_dict["write"] = "W" in row_dict["support"]
+        row_dict["create"] = "C" in row_dict["support"]
+        filetypes = [ft.lower() for ft in row_dict["file type"]]
+        for filetype in filetypes:
+            supported_filetypes[filetype] = {"extension": filetype, **row_dict}
+
+    with open(json_file, "w") as jsonfile:
+        print(f"Writing {json_file}...")
+        json.dump(supported_filetypes, jsonfile, indent=4)