From d833c14ef4b3f9375a85034cf0fb0f85a68cabb4 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sun, 27 Dec 2020 22:17:56 -0800 Subject: [PATCH] Added --sidecar exiftool, issue #303 --- osxphotos/__main__.py | 39 +++++------ osxphotos/_version.py | 2 +- osxphotos/photoinfo/_photoinfo_export.py | 87 +++++++++++++++++++++-- tests/test_cli.py | 89 ++++++++++++++++++++++-- tests/test_export_catalina_10_15_7.py | 39 +++++++++++ tests/test_exportresults.py | 6 +- 6 files changed, 225 insertions(+), 37 deletions(-) diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index faca1bd1..e41fdb14 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -1337,17 +1337,21 @@ def query( default=None, multiple=True, metavar="FORMAT", - type=click.Choice(["xmp", "json"], case_sensitive=False), - help="Create sidecar for each photo exported; valid FORMAT values: xmp, json; " - f"--sidecar json: create JSON sidecar useable by exiftool ({_EXIF_TOOL_URL}) " - "The sidecar file can be used to apply metadata to the file with exiftool, for example: " - '"exiftool -j=photoname.jpg.json photoname.jpg" ' - "The sidecar file is named in format photoname.ext.json " + type=click.Choice(["xmp", "json", "exiftool"], case_sensitive=False), + help="Create sidecar for each photo exported; valid FORMAT values: xmp, json, exiftool; " "--sidecar xmp: create XMP sidecar used by Adobe Lightroom, etc." "The sidecar file is named in format photoname.ext.xmp" "The XMP sidecar exports the following tags: Description, Title, Keywords/Tags, " "Subject (set to Keywords + PersonInImage), PersonInImage, CreateDate, ModifyDate, " "GPSLongitude. " + f"\n--sidecar json: create JSON sidecar useable by exiftool ({_EXIF_TOOL_URL}) " + "The sidecar file can be used to apply metadata to the file with exiftool, for example: " + '"exiftool -j=photoname.jpg.json photoname.jpg" ' + "The sidecar file is named in format photoname.ext.json; " + "format includes tag groups (equivalent to running 'exiftool -G -j'). " + "\n--sidecar exiftool: create JSON sidecar compatible with output of 'exiftool -j'. " + "Unlike --sidecar json, --sidecar exiftool does not export tag groups. " + "Sidecar filename is in format photoname.ext.json;" "For a list of tags exported in the JSON sidecar, see --exiftool.", ) @click.option( @@ -2098,21 +2102,6 @@ def export( ) results += export_results - # print summary results - # print(f"results_exported: {results_exported}") - # print(f"results_new: {results_new}") - # print(f"results_updated: {results_updated}") - # print(f"results_skipped: {results_skipped}") - # print(f"results_exif_updated: {results_exif_updated}") - # print(f"results_touched: {results_touched}") - # print(f"results_converted: {results_converted}") - # print(f"results_sidecar_json_written: {results_sidecar_json_written}") - # print(f"results_sidecar_json_skipped: {results_sidecar_json_skipped}") - # print(f"results_sidecar_xmp_written: {results_sidecar_xmp_written}") - # print(f"results_sidecar_xmp_skipped: {results_sidecar_xmp_skipped}") - # print(f"results_missing: {results_missing}") - # print(f"results_error: {results_error}") - if cleanup: all_files = ( results.exported @@ -2122,6 +2111,8 @@ def export( + results.converted_to_jpeg + results.sidecar_json_written + results.sidecar_json_skipped + + results.sidecar_exiftool_written + + results.sidecar_exiftool_skipped + results.sidecar_xmp_written + results.sidecar_xmp_skipped # include missing so a file that was already in export directory @@ -2763,11 +2754,13 @@ def export_photo( ) sidecar = [s.lower() for s in sidecar] - sidecar_json = sidecar_xmp = False + sidecar_json, sidecar_xmp, sidecar_exiftool = False, False, False if "json" in sidecar: sidecar_json = True if "xmp" in sidecar: sidecar_xmp = True + if "exiftool" in sidecar: + sidecar_exiftool = True # if download_missing and the photo is missing or path doesn't exist, # try to download with Photos @@ -2797,6 +2790,7 @@ def export_photo( dest_path, original_filename, sidecar_json=sidecar_json, + sidecar_exiftool=sidecar_exiftool, sidecar_xmp=sidecar_xmp, live_photo=export_live, raw_photo=export_raw, @@ -2902,6 +2896,7 @@ def export_photo( dest_path, edited_filename, sidecar_json=sidecar_json, + sidecar_exiftool=sidecar_exiftool, sidecar_xmp=sidecar_xmp, export_as_hardlink=export_as_hardlink, overwrite=overwrite, diff --git a/osxphotos/_version.py b/osxphotos/_version.py index 516ca806..b9b87992 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,5 +1,5 @@ """ version info """ -__version__ = "0.38.14" +__version__ = "0.38.15" diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index e262438d..088434c1 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -11,6 +11,7 @@ """ # TODO: should this be its own PhotoExporter class? +# TODO: the various sidecar_json, sidecar_xmp, etc args should all be collapsed to a sidecar param using a bit mask import glob import hashlib @@ -60,6 +61,8 @@ class ExportResults: converted_to_jpeg=None, sidecar_json_written=None, sidecar_json_skipped=None, + sidecar_exiftool_written=None, + sidecar_exiftool_skipped=None, sidecar_xmp_written=None, sidecar_xmp_skipped=None, missing=None, @@ -76,6 +79,8 @@ class ExportResults: self.converted_to_jpeg = converted_to_jpeg or [] self.sidecar_json_written = sidecar_json_written or [] self.sidecar_json_skipped = sidecar_json_skipped or [] + self.sidecar_exiftool_written = sidecar_exiftool_written or [] + self.sidecar_exiftool_skipped = sidecar_exiftool_skipped or [] self.sidecar_xmp_written = sidecar_xmp_written or [] self.sidecar_xmp_skipped = sidecar_xmp_skipped or [] self.missing = missing or [] @@ -95,6 +100,8 @@ class ExportResults: + self.converted_to_jpeg + self.sidecar_json_written + self.sidecar_json_skipped + + self.sidecar_exiftool_written + + self.sidecar_exiftool_skipped + self.sidecar_xmp_written + self.sidecar_xmp_skipped + self.missing @@ -116,6 +123,8 @@ class ExportResults: self.converted_to_jpeg += other.converted_to_jpeg self.sidecar_json_written += other.sidecar_json_written self.sidecar_json_skipped += other.sidecar_json_skipped + self.sidecar_exiftool_written += other.sidecar_exiftool_written + self.sidecar_exiftool_skipped += other.sidecar_exiftool_skipped self.sidecar_xmp_written += other.sidecar_xmp_written self.sidecar_xmp_skipped += other.sidecar_xmp_skipped self.missing += other.missing @@ -136,6 +145,8 @@ class ExportResults: + f",converted_to_jpeg={self.converted_to_jpeg}" + f",sidecar_json_written={self.sidecar_json_written}" + f",sidecar_json_skipped={self.sidecar_json_skipped}" + + f",sidecar_exiftool_written={self.sidecar_exiftool_written}" + + f",sidecar_exiftool_skipped={self.sidecar_exiftool_skipped}" + f",sidecar_xmp_written={self.sidecar_xmp_written}" + f",sidecar_xmp_skipped={self.sidecar_xmp_skipped}" + f",missing={self.missing}" @@ -323,6 +334,7 @@ def export( overwrite=False, increment=True, sidecar_json=False, + sidecar_exiftool=False, sidecar_xmp=False, use_photos_export=False, timeout=120, @@ -352,8 +364,10 @@ def export( overwrite: (boolean, default=False); if True will overwrite files if they alreay exist increment: (boolean, default=True); if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists - sidecar_json: (boolean, default = False); if True will also write a json sidecar with IPTC data in format readable by exiftool - sidecar filename will be dest/filename.json + sidecar_json: (boolean, default = False); if True will also write a json sidecar with data in format readable by exiftool + sidecar filename will be dest/filename.json, includes exiftool tag group names (e.g. `exiftool -G -j`) + sidecar_exiftool: (boolean, default = False); if True will also write a json sidecar with data in format readable by exiftool + sidecar filename will be dest/filename.json; does not include exiftool tag group names (e.g. `exiftool -j`) sidecar_xmp: (boolean, default = False); if True will also write a XMP sidecar with IPTC data sidecar filename will be dest/filename.xmp use_photos_export: (boolean, default=False); if True will attempt to export photo via applescript interaction with Photos @@ -382,6 +396,7 @@ def export( overwrite=overwrite, increment=increment, sidecar_json=sidecar_json, + sidecar_exiftool=sidecar_exiftool, sidecar_xmp=sidecar_xmp, use_photos_export=use_photos_export, timeout=timeout, @@ -406,6 +421,7 @@ def export2( overwrite=False, increment=True, sidecar_json=False, + sidecar_exiftool=False, sidecar_xmp=False, use_photos_export=False, timeout=120, @@ -445,8 +461,10 @@ def export2( overwrite: (boolean, default=False); if True will overwrite files if they alreay exist increment: (boolean, default=True); if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists - sidecar_json: (boolean, default = False); if True will also write a json sidecar with IPTC data in format readable by exiftool - sidecar filename will be dest/filename.json + sidecar_json: (boolean, default = False); if True will also write a json sidecar with data in format readable by exiftool + sidecar filename will be dest/filename.json; includes exiftool tag group names (e.g. `exiftool -G -j`) + sidecar_exiftool: (boolean, default = False); if True will also write a json sidecar with data in format readable by exiftool + sidecar filename will be dest/filename.json; does not include exiftool tag group names (e.g. `exiftool -j`) sidecar_xmp: (boolean, default = False); if True will also write a XMP sidecar with IPTC data sidecar filename will be dest/filename.xmp use_photos_export: (boolean, default=False); if True will attempt to export photo via applescript interaction with Photos @@ -483,6 +501,8 @@ def export2( "converted_to_jpeg", "sidecar_json_written", "sidecar_json_skipped", + "sidecar_exiftool_written", + "sidecar_exiftool_skipped", "sidecar_xmp_written", "sidecar_xmp_skipped", "missing", @@ -857,6 +877,7 @@ def export2( ) # export metadata + # TODO: repetitive code here is prime for refactoring sidecar_json_files_skipped = [] sidecar_json_files_written = [] if sidecar_json: @@ -882,7 +903,7 @@ def export2( ) ) if write_sidecar: - verbose(f"Writing exiftool JSON sidecar {sidecar_filename}") + verbose(f"Writing JSON sidecar {sidecar_filename}") sidecar_json_files_written.append(str(sidecar_filename)) if not dry_run: self._write_sidecar(sidecar_filename, sidecar_str) @@ -892,9 +913,48 @@ def export2( fileutil.file_sig(sidecar_filename), ) else: - verbose(f"Skipped up to date exiftool JSON sidecar {sidecar_filename}") + verbose(f"Skipped up to date JSON sidecar {sidecar_filename}") sidecar_json_files_skipped.append(str(sidecar_filename)) + sidecar_exiftool_files_skipped = [] + sidecar_exiftool_files_written = [] + if sidecar_exiftool: + sidecar_filename = dest.parent / pathlib.Path(f"{dest.stem}{dest.suffix}.json") + sidecar_str = self._exiftool_json_sidecar( + use_albums_as_keywords=use_albums_as_keywords, + use_persons_as_keywords=use_persons_as_keywords, + keyword_template=keyword_template, + description_template=description_template, + ignore_date_modified=ignore_date_modified, + tag_groups=False, + ) + sidecar_digest = hexdigest(sidecar_str) + old_sidecar_digest, sidecar_sig = export_db.get_sidecar_for_file( + sidecar_filename + ) + write_sidecar = ( + not update + or (update and not sidecar_filename.exists()) + or ( + update + and (sidecar_digest != old_sidecar_digest) + or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig) + ) + ) + if write_sidecar: + verbose(f"Writing exiftool sidecar {sidecar_filename}") + sidecar_exiftool_files_written.append(str(sidecar_filename)) + if not dry_run: + self._write_sidecar(sidecar_filename, sidecar_str) + export_db.set_sidecar_for_file( + sidecar_filename, + sidecar_digest, + fileutil.file_sig(sidecar_filename), + ) + else: + verbose(f"Skipped up to date exiftool sidecar {sidecar_filename}") + sidecar_exiftool_files_skipped.append(str(sidecar_filename)) + sidecar_xmp_files_skipped = [] sidecar_xmp_files_written = [] if sidecar_xmp: @@ -1051,6 +1111,8 @@ def export2( converted_to_jpeg=converted_to_jpeg_files, sidecar_json_written=sidecar_json_files_written, sidecar_json_skipped=sidecar_json_files_skipped, + sidecar_exiftool_written=sidecar_exiftool_files_written, + sidecar_exiftool_skipped=sidecar_exiftool_files_skipped, sidecar_xmp_written=sidecar_xmp_files_written, sidecar_xmp_skipped=sidecar_xmp_files_skipped, error=errors, @@ -1233,6 +1295,8 @@ def _export_photo( converted_to_jpeg=converted_to_jpeg_files, sidecar_json_written=[], sidecar_json_skipped=[], + sidecar_exiftool_written=[], + sidecar_exiftool_skipped=[], sidecar_xmp_written=[], sidecar_xmp_skipped=[], missing=[], @@ -1482,6 +1546,7 @@ def _exiftool_json_sidecar( keyword_template=None, description_template=None, ignore_date_modified=False, + tag_groups=True, ): """Return dict of EXIF details for building exiftool JSON sidecar or sending commands to ExifTool. Does not include all the EXIF fields as those are likely already in the image. @@ -1492,6 +1557,7 @@ def _exiftool_json_sidecar( keyword_template: (list of strings); list of template strings to render as keywords description_template: (list of strings); list of template strings to render for the description ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set + tag_groups: if True, tags are in form Group:TagName, e.g. IPTC:Keywords, otherwise group name is omitted, e.g. Keywords Returns: dict with exiftool tags / values @@ -1524,6 +1590,15 @@ def _exiftool_json_sidecar( description_template=description_template, ignore_date_modified=ignore_date_modified, ) + + if not tag_groups: + # strip tag groups + exif_new = {} + for k, v in exif.items(): + k = re.sub(r".*:", "", k) + exif_new[k] = v + exif = exif_new + return json.dumps([exif]) diff --git a/tests/test_cli.py b/tests/test_cli.py index ebc859cd..29b44071 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2003,6 +2003,38 @@ def test_export_sidecar(): assert sorted(files) == sorted(CLI_EXPORT_SIDECAR_FILENAMES) +def test_export_sidecar_exiftool(): + """ test --sidecar exiftool """ + import glob + import os + import os.path + import osxphotos + + from osxphotos.__main__ import cli + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + [ + "export", + "--db", + os.path.join(cwd, CLI_PHOTOS_DB), + ".", + "--sidecar=exiftool", + "--sidecar=xmp", + f"--uuid={CLI_EXPORT_UUID}", + "-V", + ], + ) + assert result.exit_code == 0 + assert "Writing exiftool sidecar" in result.output + files = glob.glob("*.*") + assert sorted(files) == sorted(CLI_EXPORT_SIDECAR_FILENAMES) + + def test_export_sidecar_templates(): import json import os @@ -2045,6 +2077,49 @@ def test_export_sidecar_templates(): ) +def test_export_sidecar_templates_exiftool(): + """ test --sidecar exiftool with templates """ + import json + import os + import os.path + import osxphotos + + from osxphotos.__main__ import cli + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + [ + "export", + "--db", + os.path.join(cwd, PHOTOS_DB_15_5), + ".", + "--sidecar=exiftool", + f"--uuid={CLI_UUID_DICT_15_5['template']}", + "-V", + "--keyword-template", + "{person}", + "--description-template", + "{descr} {person} {keyword} {album}", + ], + ) + assert result.exit_code == 0 + assert os.path.isfile(CLI_TEMPLATE_SIDECAR_FILENAME) + with open(CLI_TEMPLATE_SIDECAR_FILENAME, "r") as jsonfile: + exifdata = json.load(jsonfile) + assert ( + exifdata[0]["Description"] + == "Girls with pumpkins Katie, Suzy Kids Pumpkin Farm, Test Album" + ) + assert ( + exifdata[0]["ImageDescription"] + == "Girls with pumpkins Katie, Suzy Kids Pumpkin Farm, Test Album" + ) + + def test_export_sidecar_update(): """ test sidecar don't update if not changed and do update if changed """ import datetime @@ -2075,7 +2150,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Writing XMP sidecar" in result.output - assert "Writing exiftool JSON sidecar" in result.output + assert "Writing JSON sidecar" in result.output # delete a sidecar file and run update fileutil = FileUtil() @@ -2097,7 +2172,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Skipped up to date XMP sidecar" in result.output - assert "Writing exiftool JSON sidecar" in result.output + assert "Writing JSON sidecar" in result.output # run update again, no sidecar files should update result = runner.invoke( @@ -2116,7 +2191,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Skipped up to date XMP sidecar" in result.output - assert "Skipped up to date exiftool JSON sidecar" in result.output + assert "Skipped up to date JSON sidecar" in result.output # touch a file and export again ts = datetime.datetime.now().timestamp() + 1000 @@ -2138,7 +2213,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Writing XMP sidecar" in result.output - assert "Skipped up to date exiftool JSON sidecar" in result.output + assert "Skipped up to date JSON sidecar" in result.output # run update again, no sidecar files should update result = runner.invoke( @@ -2157,7 +2232,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Skipped up to date XMP sidecar" in result.output - assert "Skipped up to date exiftool JSON sidecar" in result.output + assert "Skipped up to date JSON sidecar" in result.output # run update again with updated metadata, forcing update result = runner.invoke( @@ -2178,7 +2253,7 @@ def test_export_sidecar_update(): ) assert result.exit_code == 0 assert "Writing XMP sidecar" in result.output - assert "Writing exiftool JSON sidecar" in result.output + assert "Writing JSON sidecar" in result.output def test_export_live(): @@ -4442,7 +4517,7 @@ def test_save_load_config(): ], ) assert result.exit_code == 0 - assert "Writing exiftool JSON sidecar" in result.output + assert "Writing JSON sidecar" in result.output assert "Writing XMP sidecar" not in result.output diff --git a/tests/test_export_catalina_10_15_7.py b/tests/test_export_catalina_10_15_7.py index b24d3610..9e41b23e 100644 --- a/tests/test_export_catalina_10_15_7.py +++ b/tests/test_export_catalina_10_15_7.py @@ -91,6 +91,21 @@ EXIF_JSON_EXPECTED = """ "EXIF:ModifyDate": "2019:07:27 17:33:28"}] """ +EXIFTOOL_SIDECAR_EXPECTED = """ + [{"ImageDescription": "Bride Wedding day", + "Description": "Bride Wedding day", + "TagsList": ["Maria", "wedding"], + "Keywords": ["Maria", "wedding"], + "PersonInImage": ["Maria"], + "Subject": ["wedding", "Maria"], + "DateTimeOriginal": "2019:04:15 14:40:24", + "CreateDate": "2019:04:15 14:40:24", + "OffsetTimeOriginal": "-04:00", + "DateCreated": "2019:04:15", + "TimeCreated": "14:40:24-04:00", + "ModifyDate": "2019:07:27 17:33:28"}] + """ + EXIF_JSON_EXPECTED_IGNORE_DATE_MODIFIED = """ [{"EXIF:ImageDescription": "Bride Wedding day", "XMP:Description": "Bride Wedding day", @@ -658,6 +673,30 @@ def test_exiftool_json_sidecar_use_albums_keyword(photosdb): assert json_got[k] == v +def test_exiftool_sidecar(photosdb): + import json + + photos = photosdb.photos(uuid=[EXIF_JSON_UUID]) + + json_expected = json.loads(EXIFTOOL_SIDECAR_EXPECTED)[0] + + json_got = photos[0]._exiftool_json_sidecar(tag_groups=False) + json_got = json.loads(json_got)[0] + + # some gymnastics to account for different sort order in different pythons + for k, v in json_got.items(): + if type(v) in (list, tuple): + assert sorted(json_expected[k]) == sorted(v) + else: + assert json_expected[k] == v + + for k, v in json_expected.items(): + if type(v) in (list, tuple): + assert sorted(json_got[k]) == sorted(v) + else: + assert json_got[k] == v + + @pytest.mark.skipif(exiftool is None, reason="exiftool not installed") def test_xmp_sidecar_is_valid(tmp_path, photosdb): """ validate XMP sidecar file with exiftool """ diff --git a/tests/test_exportresults.py b/tests/test_exportresults.py index fa3e2f84..fa93ca64 100644 --- a/tests/test_exportresults.py +++ b/tests/test_exportresults.py @@ -13,6 +13,8 @@ EXPORT_RESULT_ATTRIBUTES = [ "converted_to_jpeg", "sidecar_json_written", "sidecar_json_skipped", + "sidecar_exiftool_written", + "sidecar_exiftool_skipped", "sidecar_xmp_written", "sidecar_xmp_skipped", "missing", @@ -33,6 +35,8 @@ def test_exportresults_init(): assert results.converted_to_jpeg == [] assert results.sidecar_json_written == [] assert results.sidecar_json_skipped == [] + assert results.sidecar_exiftool_written == [] + assert results.sidecar_exiftool_skipped == [] assert results.sidecar_xmp_written == [] assert results.sidecar_xmp_skipped == [] assert results.missing == [] @@ -90,6 +94,6 @@ def test_str(): results = ExportResults() assert ( str(results) - == "ExportResults(exported=[],new=[],updated=[],skipped=[],exif_updated=[],touched=[],converted_to_jpeg=[],sidecar_json_written=[],sidecar_json_skipped=[],sidecar_xmp_written=[],sidecar_xmp_skipped=[],missing=[],error=[],exiftool_warning=[],exiftool_error=[])" + == "ExportResults(exported=[],new=[],updated=[],skipped=[],exif_updated=[],touched=[],converted_to_jpeg=[],sidecar_json_written=[],sidecar_json_skipped=[],sidecar_exiftool_written=[],sidecar_exiftool_skipped=[],sidecar_xmp_written=[],sidecar_xmp_skipped=[],missing=[],error=[],exiftool_warning=[],exiftool_error=[])" )