Added exiftool signature to JSON output, issue #303

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.38.18](https://github.com/RhetTbull/osxphotos/compare/v0.38.17...v0.38.18)
+
+> 28 December 2020
+
+- Added --exiftool-merge-keywords/persons, issue #299, #292 [`b1cb99f`](https://github.com/RhetTbull/osxphotos/commit/b1cb99f83f55128a314d265d4588134cb79026c6)
+
+#### [v0.38.17](https://github.com/RhetTbull/osxphotos/compare/v0.38.16...v0.38.17)
+
+> 28 December 2020
+
+- Added --sidecar-drop-ext, issue #291 [`dce002c`](https://github.com/RhetTbull/osxphotos/commit/dce002cdfe12fa5fa4ada4d5097828a5375c2ecd)
+- Updated Template Substitution table [`7bd189e`](https://github.com/RhetTbull/osxphotos/commit/7bd189e9b22a2ad5a8a80deb7cb93c61be37c771)
+
 #### [v0.38.16](https://github.com/RhetTbull/osxphotos/compare/v0.38.15...v0.38.16)
 
 > 28 December 2020

README.md

@@ -373,6 +373,12 @@ Options:
                                   may be specified by repeating the option,
                                   e.g. --exiftool-option '-m' --exiftool-
                                   option '-F'.
+  --exiftool-merge-keywords       Merge any keywords found in the original
+                                  file with keywords used for '--exiftool' and
+                                  '--sidecar'.
+  --exiftool-merge-persons        Merge any persons found in the original file
+                                  with persons used for '--exiftool' and '--
+                                  sidecar'.
   --ignore-date-modified          If used with --exiftool or --sidecar, will
                                   ignore the photo modification date and set
                                   EXIF:ModifyDate to EXIF:DateTimeOriginal;

osxphotos/__main__.py

@@ -1394,6 +1394,16 @@ def query(
     "More than one option may be specified by repeating the option, e.g. "
     "--exiftool-option '-m' --exiftool-option '-F'. ",
 )
+@click.option(
+    "--exiftool-merge-keywords",
+    is_flag=True,
+    help="Merge any keywords found in the original file with keywords used for '--exiftool' and '--sidecar'.",
+)
+@click.option(
+    "--exiftool-merge-persons",
+    is_flag=True,
+    help="Merge any persons found in the original file with persons used for '--exiftool' and '--sidecar'.",
+)
 @click.option(
     "--ignore-date-modified",
     is_flag=True,
@@ -1592,6 +1602,8 @@ def export(
     dest,
     exiftool,
     exiftool_option,
+    exiftool_merge_keywords,
+    exiftool_merge_persons,
     ignore_date_modified,
     portrait,
     not_portrait,
@@ -1714,6 +1726,7 @@ def export(
         convert_to_jpeg = cfg.convert_to_jpeg
         jpeg_quality = cfg.jpeg_quality
         sidecar = cfg.sidecar
+        sidecar_drop_ext = cfg.sidecar_drop_ext
         only_photos = cfg.only_photos
         only_movies = cfg.only_movies
         burst = cfg.burst
@@ -1722,6 +1735,9 @@ def export(
         not_live = cfg.not_live
         download_missing = cfg.download_missing
         exiftool = cfg.exiftool
+        exiftool_option = cfg.exiftool_option
+        exiftool_merge_keywords = cfg.exiftool_merge_keywords
+        exiftool_merge_persons = cfg.exiftool_merge_persons
         ignore_date_modified = cfg.ignore_date_modified
         portrait = cfg.portrait
         not_portrait = cfg.not_portrait
@@ -1795,6 +1811,8 @@ def export(
         ("jpeg_quality", ("convert_to_jpeg")),
         ("ignore_signature", ("update")),
         ("exiftool_option", ("exiftool")),
+        ("exiftool_merge_keywords", ("exiftool", "sidecar")),
+        ("exiftool_merge_persons", ("exiftool", "sidecar")),
     ]
     try:
         cfg.validate(exclusive=exclusive_options, dependent=dependent_options, cli=True)
@@ -2058,6 +2076,8 @@ def export(
                     export_live=export_live,
                     download_missing=download_missing,
                     exiftool=exiftool,
+                    exiftool_merge_keywords=exiftool_merge_keywords,
+                    exiftool_merge_persons=exiftool_merge_persons,
                     directory=directory,
                     filename_template=filename_template,
                     export_raw=export_raw,
@@ -2107,6 +2127,8 @@ def export(
                         export_live=export_live,
                         download_missing=download_missing,
                         exiftool=exiftool,
+                        exiftool_merge_keywords=exiftool_merge_keywords,
+                        exiftool_merge_persons=exiftool_merge_persons,
                         directory=directory,
                         filename_template=filename_template,
                         export_raw=export_raw,
@@ -2640,6 +2662,8 @@ def export_photo(
     export_live=None,
     download_missing=None,
     exiftool=None,
+    exiftool_merge_keywords=False,
+    exiftool_merge_persons=False,
     directory=None,
     filename_template=None,
     export_raw=None,
@@ -2694,6 +2718,8 @@ def export_photo(
         jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0.  A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
         ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
         exiftool_option: optional list flags (e.g. ["-m", "-F"]) to pass to exiftool
+        exiftool_merge_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
+        exiftool_merge_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
 
     Returns:
         list of path(s) of exported photo or None if photo was missing
@@ -2826,6 +2852,8 @@ def export_photo(
                             overwrite=overwrite,
                             use_photos_export=use_photos_export,
                             exiftool=exiftool,
+                            merge_exif_keywords=exiftool_merge_keywords,
+                            merge_exif_persons=exiftool_merge_persons,
                             use_albums_as_keywords=album_keyword,
                             use_persons_as_keywords=person_keyword,
                             keyword_template=keyword_template,
@@ -2930,6 +2958,8 @@ def export_photo(
                             edited=True,
                             use_photos_export=use_photos_export,
                             exiftool=exiftool,
+                            merge_exif_keywords=exiftool_merge_keywords,
+                            merge_exif_persons=exiftool_merge_persons,
                             use_albums_as_keywords=album_keyword,
                             use_persons_as_keywords=person_keyword,
                             keyword_template=keyword_template,

osxphotos/_version.py

@@ -1,5 +1,5 @@
 """ version info """
 
-__version__ = "0.38.17"
+__version__ = "0.38.19"
 
 

osxphotos/photoinfo/_photoinfo_export.py

@@ -5,6 +5,8 @@
     _export_photo
     _write_exif_data
     _exiftool_json_sidecar
+    _get_exif_keywords
+    _get_exif_persons
     _exiftool_dict
     _xmp_sidecar
     _write_sidecar
@@ -450,6 +452,8 @@ def export2(
     use_photokit=False,
     verbose=None,
     exiftool_flags=None,
+    merge_exif_keywords=False,
+    merge_exif_persons=False,
 ):
     """export photo, like export but with update and dry_run options
     dest: must be valid destination path or exception raised
@@ -499,6 +503,8 @@ def export2(
     ignore_date_modified: for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
     verbose: optional callable function to use for printing verbose text during processing; if None (default), does not print output.
     exiftool_flags: optional list of flags to pass to exiftool when using exiftool option, e.g ["-m", "-F"]
+    merge_exif_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
+    merge_exif_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
 
     Returns: ExportResults class 
         ExportResults has attributes: 
@@ -904,6 +910,9 @@ def export2(
             keyword_template=keyword_template,
             description_template=description_template,
             ignore_date_modified=ignore_date_modified,
+            merge_exif_keywords=merge_exif_keywords,
+            merge_exif_persons=merge_exif_persons,
+            filename=dest.name,
         )
         sidecars.append(
             (
@@ -924,6 +933,9 @@ def export2(
             description_template=description_template,
             ignore_date_modified=ignore_date_modified,
             tag_groups=False,
+            merge_exif_keywords=merge_exif_keywords,
+            merge_exif_persons=merge_exif_persons,
+            filename=dest.name,
         )
         sidecars.append(
             (
@@ -1012,6 +1024,8 @@ def export2(
                         keyword_template=keyword_template,
                         description_template=description_template,
                         ignore_date_modified=ignore_date_modified,
+                        merge_exif_keywords=merge_exif_keywords,
+                        merge_exif_persons=merge_exif_persons,
                     )
                 )[0]
                 if old_data != current_data:
@@ -1030,6 +1044,8 @@ def export2(
                         description_template=description_template,
                         ignore_date_modified=ignore_date_modified,
                         flags=exiftool_flags,
+                        merge_exif_keywords=merge_exif_keywords,
+                        merge_exif_persons=merge_exif_persons,
                     )
                     if warning_:
                         exiftool_warning.append((exported_file, warning_))
@@ -1045,6 +1061,8 @@ def export2(
                         keyword_template=keyword_template,
                         description_template=description_template,
                         ignore_date_modified=ignore_date_modified,
+                        merge_exif_keywords=merge_exif_keywords,
+                        merge_exif_persons=merge_exif_persons,
                     ),
                 )
                 export_db.set_stat_exif_for_file(
@@ -1065,6 +1083,8 @@ def export2(
                     description_template=description_template,
                     ignore_date_modified=ignore_date_modified,
                     flags=exiftool_flags,
+                    merge_exif_keywords=merge_exif_keywords,
+                    merge_exif_persons=merge_exif_persons,
                 )
                 if warning_:
                     exiftool_warning.append((exported_file, warning_))
@@ -1080,6 +1100,8 @@ def export2(
                     keyword_template=keyword_template,
                     description_template=description_template,
                     ignore_date_modified=ignore_date_modified,
+                    merge_exif_keywords=merge_exif_keywords,
+                    merge_exif_persons=merge_exif_persons,
                 ),
             )
             export_db.set_stat_exif_for_file(
@@ -1308,6 +1330,8 @@ def _write_exif_data(
     description_template=None,
     ignore_date_modified=False,
     flags=None,
+    merge_exif_keywords=False,
+    merge_exif_persons=False,
 ):
     """write exif data to image file at filepath
 
@@ -1330,6 +1354,8 @@ def _write_exif_data(
         keyword_template=keyword_template,
         description_template=description_template,
         ignore_date_modified=ignore_date_modified,
+        merge_exif_keywords=merge_exif_keywords,
+        merge_exif_persons=merge_exif_persons,
     )
 
     with ExifTool(filepath, flags=flags) as exiftool:
@@ -1349,16 +1375,22 @@ def _exiftool_dict(
     keyword_template=None,
     description_template=None,
     ignore_date_modified=False,
+    merge_exif_keywords=False,
+    merge_exif_persons=False,
+    filename=None,
 ):
     """Return dict of EXIF details for building exiftool JSON sidecar or sending commands to ExifTool.
         Does not include all the EXIF fields as those are likely already in the image.
 
     Args:
+        filename: name of source image file (without path); if not None, exiftool JSON signature will be included; if None, signature will not be included
         use_albums_as_keywords: treat album names as keywords
         use_persons_as_keywords: treat person names as keywords
         keyword_template: (list of strings); list of template strings to render as keywords
         description_template: (list of strings); list of template strings to render for the description
         ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
+        merge_exif_keywords: merge keywords in the file's exif metadata (requires exiftool)
+        merge_exif_persons: merge persons in the file's exif metadata (requires exiftool)
 
     Returns: dict with exiftool tags / values
 
@@ -1385,7 +1417,16 @@ def _exiftool_dict(
         UserData:GPSCoordinates
     """
 
-    exif = {}
+    exif = (
+        {
+            "SourceFile": filename,
+            "ExifTool:ExifToolVersion": "12.00",
+            "File:FileName": filename,
+        }
+        if filename is not None
+        else {}
+    )
+
     if description_template is not None:
         rendered = self.render_template(
             description_template, expand_inplace=True, inplace_sep=", "
@@ -1401,13 +1442,19 @@ def _exiftool_dict(
         exif["XMP:Title"] = self.title
 
     keyword_list = []
+    if merge_exif_keywords:
+        keyword_list.extend(self._get_exif_keywords())
+
     if self.keywords:
         keyword_list.extend(self.keywords)
 
     person_list = []
+    if merge_exif_persons:
+        person_list.extend(self._get_exif_persons())
+
     if self.persons:
         # filter out _UNKNOWN_PERSON
-        person_list = [p for p in self.persons if p != _UNKNOWN_PERSON]
+        person_list.extend([p for p in self.persons if p != _UNKNOWN_PERSON])
 
     if use_persons_as_keywords and person_list:
         keyword_list.extend(person_list)
@@ -1534,6 +1581,39 @@ def _exiftool_dict(
     return exif
 
 
+def _get_exif_keywords(self):
+    """ returns list of keywords found in the file's exif metadata """
+    keywords = []
+    exif = self.exiftool
+    if exif:
+        exifdict = exif.asdict()
+        for field in ["IPTC:Keywords", "XMP:TagsList", "XMP:Subject"]:
+            try:
+                kw = exifdict[field]
+                if kw and type(kw) != list:
+                    kw = [kw]
+                keywords.extend(kw)
+            except KeyError:
+                pass
+    return keywords
+
+
+def _get_exif_persons(self):
+    """ returns list of persons found in the file's exif metadata """
+    persons = []
+    exif = self.exiftool
+    if exif:
+        exifdict = exif.asdict()
+        try:
+            p = exifdict["XMP:PersonInImage"]
+            if p and type(p) != list:
+                p = [p]
+            persons.extend(p)
+        except KeyError:
+            pass
+    return persons
+
+
 def _exiftool_json_sidecar(
     self,
     use_albums_as_keywords=False,
@@ -1542,6 +1622,9 @@ def _exiftool_json_sidecar(
     description_template=None,
     ignore_date_modified=False,
     tag_groups=True,
+    merge_exif_keywords=False,
+    merge_exif_persons=False,
+    filename=None,
 ):
     """Return dict of EXIF details for building exiftool JSON sidecar or sending commands to ExifTool.
         Does not include all the EXIF fields as those are likely already in the image.
@@ -1553,6 +1636,9 @@ def _exiftool_json_sidecar(
         description_template: (list of strings); list of template strings to render for the description
         ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
         tag_groups: if True, tags are in form Group:TagName, e.g. IPTC:Keywords, otherwise group name is omitted, e.g. Keywords
+        merge_exif_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
+        merge_exif_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
+        filename: filename of the destination image file for including in exiftool signature in JSON sidecar
 
     Returns: dict with exiftool tags / values
 
@@ -1584,6 +1670,9 @@ def _exiftool_json_sidecar(
         keyword_template=keyword_template,
         description_template=description_template,
         ignore_date_modified=ignore_date_modified,
+        merge_exif_keywords=merge_exif_keywords,
+        merge_exif_persons=merge_exif_persons,
+        filename=filename,
     )
 
     if not tag_groups:
@@ -1604,12 +1693,17 @@ def _xmp_sidecar(
     keyword_template=None,
     description_template=None,
     extension=None,
+    merge_exif_keywords=False,
+    merge_exif_persons=False,
 ):
     """returns string for XMP sidecar
     use_albums_as_keywords: treat album names as keywords
     use_persons_as_keywords: treat person names as keywords
     keyword_template: (list of strings); list of template strings to render as keywords
-    description_template: string; optional template string that will be rendered for use as photo description"""
+    description_template: string; optional template string that will be rendered for use as photo description
+    merge_exif_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
+    merge_exif_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
+    """
 
     xmp_template = Template(filename=os.path.join(_TEMPLATE_DIR, _XMP_TEMPLATE_NAME))
 
@@ -1626,6 +1720,9 @@ def _xmp_sidecar(
         description = self.description if self.description is not None else ""
 
     keyword_list = []
+    if merge_exif_keywords:
+        keyword_list.extend(self._get_exif_keywords())
+
     if self.keywords:
         keyword_list.extend(self.keywords)
 
@@ -1633,9 +1730,12 @@ def _xmp_sidecar(
     # good candidate for pulling out in a function
 
     person_list = []
+    if merge_exif_persons:
+        person_list.extend(self._get_exif_persons())
+
     if self.persons:
         # filter out _UNKNOWN_PERSON
-        person_list = [p for p in self.persons if p != _UNKNOWN_PERSON]
+        person_list.extend([p for p in self.persons if p != _UNKNOWN_PERSON])
 
     if use_persons_as_keywords and person_list:
         keyword_list.extend(person_list)

osxphotos/photoinfo/photoinfo.py

@@ -21,11 +21,11 @@ from .._constants import (
     _PHOTOS_4_ALBUM_KIND,
     _PHOTOS_4_ROOT_FOLDER,
     _PHOTOS_4_VERSION,
-    _PHOTOS_5_VERSION,
     _PHOTOS_5_ALBUM_KIND,
     _PHOTOS_5_IMPORT_SESSION_ALBUM_KIND,
     _PHOTOS_5_SHARED_ALBUM_KIND,
     _PHOTOS_5_SHARED_PHOTO_PATH,
+    _PHOTOS_5_VERSION,
 )
 from ..albuminfo import AlbumInfo, ImportInfo
 from ..personinfo import FaceInfo, PersonInfo
@@ -56,6 +56,8 @@ class PhotoInfo:
         _export_photo,
         _exiftool_dict,
         _exiftool_json_sidecar,
+        _get_exif_keywords,
+        _get_exif_persons,
         _write_exif_data,
         _write_sidecar,
         _xmp_sidecar,

tests/test_cli.py

@@ -367,6 +367,29 @@ CLI_EXIFTOOL = {
     }
 }
 
+CLI_EXIFTOOL_MERGE = {
+    "1EB2B765-0765-43BA-A90C-0D0580E6172C": {
+        "File:FileName": "Pumpkins3.jpg",
+        "IPTC:Keywords": "Kids",
+        "XMP:TagsList": "Kids",
+        "EXIF:ImageDescription": "Kids in pumpkin field",
+        "XMP:Description": "Kids in pumpkin field",
+        "XMP:PersonInImage": ["Katie", "Suzy", "Tim"],
+        "XMP:Subject": "Kids",
+    },
+    "D79B8D77-BFFC-460B-9312-034F2877D35B": {
+        "File:FileName": "Pumkins2.jpg",
+        "XMP:Title": "I found one!",
+        "EXIF:ImageDescription": "Girl holding pumpkin",
+        "XMP:Description": "Girl holding pumpkin",
+        "XMP:PersonInImage": "Katie",
+        "IPTC:Keywords": ["Kids", "keyword1", "keyword2", "subject1", "tagslist1"],
+        "XMP:TagsList": ["Kids", "keyword1", "keyword2", "subject1", "tagslist1"],
+        "XMP:Subject": ["Kids", "keyword1", "keyword2", "subject1", "tagslist1"],
+    },
+}
+
+
 CLI_EXIFTOOL_QUICKTIME = {
     "35329C57-B963-48D6-BB75-6AFF9370CBBC": {
         "File:FileName": "Jellyfish.MOV",
@@ -1216,6 +1239,96 @@ def test_export_exiftool_option():
         assert "exiftool warning" not in result.output
 
 
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_exiftool_merge():
+    """ test --exiftool-merge-keywords and --exiftool-merge-persons """
+    import glob
+    import os
+    import os.path
+    from osxphotos.__main__ import export
+    from osxphotos.exiftool import ExifTool
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        for uuid in CLI_EXIFTOOL_MERGE:
+            result = runner.invoke(
+                export,
+                [
+                    os.path.join(cwd, PHOTOS_DB_15_7),
+                    ".",
+                    "-V",
+                    "--exiftool",
+                    "--uuid",
+                    f"{uuid}",
+                    "--exiftool-merge-keywords",
+                    "--exiftool-merge-persons",
+                ],
+            )
+            assert result.exit_code == 0
+            files = glob.glob("*")
+            assert CLI_EXIFTOOL_MERGE[uuid]["File:FileName"] in files
+
+            exif = ExifTool(CLI_EXIFTOOL_MERGE[uuid]["File:FileName"]).asdict()
+            for key in CLI_EXIFTOOL_MERGE[uuid]:
+                if type(exif[key]) == list:
+                    assert sorted(exif[key]) == sorted(CLI_EXIFTOOL_MERGE[uuid][key])
+                else:
+                    assert exif[key] == CLI_EXIFTOOL_MERGE[uuid][key]
+
+
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_exiftool_merge_sidecar():
+    """ test --exiftool-merge-keywords and --exiftool-merge-persons with --sidecar """
+    import glob
+    import json
+    import os
+    import os.path
+    from osxphotos.__main__ import export
+    from osxphotos.exiftool import ExifTool
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        for uuid in CLI_EXIFTOOL_MERGE:
+            result = runner.invoke(
+                export,
+                [
+                    os.path.join(cwd, PHOTOS_DB_15_7),
+                    ".",
+                    "-V",
+                    "--sidecar",
+                    "json",
+                    "--uuid",
+                    f"{uuid}",
+                    "--exiftool-merge-keywords",
+                    "--exiftool-merge-persons",
+                ],
+            )
+            assert result.exit_code == 0
+            files = glob.glob("*")
+            json_file = f"{CLI_EXIFTOOL_MERGE[uuid]['File:FileName']}.json"
+            assert json_file in files
+
+            with open(json_file, "r") as fp:
+                exif = json.load(fp)[0]
+
+            for key in CLI_EXIFTOOL_MERGE[uuid]:
+                if key == "File:FileName":
+                    continue
+                if type(exif[key]) == list:
+                    expected = (
+                        CLI_EXIFTOOL_MERGE[uuid][key]
+                        if type(CLI_EXIFTOOL_MERGE[uuid][key]) == list
+                        else [CLI_EXIFTOOL_MERGE[uuid][key]]
+                    )
+                    assert sorted(exif[key]) == sorted(expected)
+                else:
+                    assert exif[key] == CLI_EXIFTOOL_MERGE[uuid][key]
+
+
 def test_export_edited_suffix():
     """ test export with --edited-suffix """
     import glob
@@ -3269,9 +3382,12 @@ def test_export_sidecar_keyword_template():
 
         json_expected = json.loads(
             """
-            [{"EXIF:ImageDescription": "Girl holding pumpkin", 
+            [{"SourceFile": "Pumkins2.jpg",
+            "ExifTool:ExifToolVersion": "12.00",
+            "File:FileName": "Pumkins2.jpg",
+            "EXIF:ImageDescription": "Girl holding pumpkin", 
             "XMP:Description": "Girl holding pumpkin", 
-            "XMP:Title": "I found one!", 
+            "XMP:Title": "I found one!",
             "XMP:TagsList": ["Kids", "Multi Keyword", "Pumpkin Farm", "Test Album"], 
             "IPTC:Keywords": ["Kids", "Multi Keyword", "Pumpkin Farm", "Test Album"], 
             "XMP:PersonInImage": ["Katie"],