remove duplicate keywords with --exiftool and --sidecar, closes #294

osxphotos/_version.py

@@ -1,5 +1,5 @@
 """ version info """
 
-__version__ = "0.38.7"
+__version__ = "0.38.8"
 
 

osxphotos/photoinfo/_photoinfo_export.py

@@ -1341,13 +1341,13 @@ def _exiftool_dict(
     person_list = []
     if self.persons:
         # filter out _UNKNOWN_PERSON
-        person_list = sorted([p for p in self.persons if p != _UNKNOWN_PERSON])
+        person_list = [p for p in self.persons if p != _UNKNOWN_PERSON]
 
     if use_persons_as_keywords and person_list:
-        keyword_list.extend(sorted(person_list))
+        keyword_list.extend(person_list)
 
     if use_albums_as_keywords and self.albums:
-        keyword_list.extend(sorted(self.albums))
+        keyword_list.extend(self.albums)
 
     if keyword_template:
         rendered_keywords = []
@@ -1382,16 +1382,19 @@ def _exiftool_dict(
         keyword_list.extend(rendered_keywords)
 
     if keyword_list:
+        # remove duplicates
+        keyword_list = sorted(list(set(keyword_list)))
         exif["XMP:TagsList"] = keyword_list.copy()
         exif["IPTC:Keywords"] = keyword_list.copy()
 
     if person_list:
+        person_list = sorted(list(set(person_list)))
         exif["XMP:PersonInImage"] = person_list.copy()
 
     if self.keywords or person_list:
         # Photos puts both keywords and persons in Subject when using "Export IPTC as XMP"
         # only use Photos' keywords for subject (e.g. don't include template values)
-        exif["XMP:Subject"] = self.keywords.copy() + person_list.copy()
+        exif["XMP:Subject"] = sorted(list(set(self.keywords + person_list)))
 
     # if self.favorite():
     #     exif["Rating"] = 5
@@ -1460,13 +1463,12 @@ def _exiftool_dict(
         date_utc = datetime_tz_to_utc(date)
         creationdate = date_utc.strftime("%Y:%m:%d %H:%M:%S")
         exif["QuickTime:CreateDate"] = creationdate
-        if self.date_modified is not None and not ignore_date_modified:
+        if self.date_modified is None or ignore_date_modified:
+            exif["QuickTime:ModifyDate"] = creationdate
+        else:
             exif["QuickTime:ModifyDate"] = datetime_tz_to_utc(
                 self.date_modified
             ).strftime("%Y:%m:%d %H:%M:%S")
-        else:
-            exif["QuickTime:ModifyDate"] = creationdate
-
     return exif
 
 
@@ -1604,6 +1606,15 @@ def _xmp_sidecar(
         # Photos puts both keywords and persons in Subject when using "Export IPTC as XMP"
         subject_list = list(self.keywords) + person_list
 
+    # remove duplicates
+    # sorted mainly to make testing the XMP file easier
+    if keyword_list:
+        keyword_list = sorted(list(set(keyword_list)))
+    if subject_list:
+        subject_list = sorted(list(set(subject_list)))
+    if person_list:
+        person_list = sorted(list(set(person_list)))
+
     xmp_str = xmp_template.render(
         photo=self,
         description=description,

tests/Test-10.15.7.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
 	<key>pid</key>
-	<integer>19275</integer>
+	<integer>55247</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/test_catalina_10_15_7.py

@@ -35,6 +35,7 @@ KEYWORDS = [
     "United Kingdom",
     "foo/bar",
     "Travel",
+    "Maria",
 ]
 # Photos 5 includes blank person for detected face
 PERSONS = ["Katie", "Suzy", "Maria", _UNKNOWN_PERSON]
@@ -60,6 +61,7 @@ KEYWORDS_DICT = {
     "United Kingdom": 1,
     "foo/bar": 1,
     "Travel": 2,
+    "Maria": 1,
 }
 PERSONS_DICT = {"Katie": 3, "Suzy": 2, "Maria": 2, _UNKNOWN_PERSON: 1}
 ALBUM_DICT = {
@@ -339,7 +341,7 @@ def test_attributes_2(photosdb):
     photos = photosdb.photos(uuid=[UUID_DICT["has_adjustments"]])
     assert len(photos) == 1
     p = photos[0]
-    assert p.keywords == ["wedding"]
+    assert sorted(p.keywords) == ["Maria", "wedding"]
     assert p.original_filename == "wedding.jpg"
     assert p.filename == "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg"
     assert p.date == datetime.datetime(

tests/test_cli.py

@@ -407,6 +407,10 @@ CLI_EXIFTOOL_IGNORE_DATE_MODIFIED = {
 
 CLI_EXIFTOOL_ERROR = ["E2078879-A29C-4D6F-BACB-E3BBE6C3EB91"]
 
+CLI_EXIFTOOL_DUPLICATE_KEYWORDS = {
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51": "wedding.jpg"
+}
+
 LABELS_JSON = {
     "labels": {
         "Plant": 7,
@@ -1017,6 +1021,9 @@ def test_export_exiftool():
 
             exif = ExifTool(CLI_EXIFTOOL[uuid]["File:FileName"]).asdict()
             for key in CLI_EXIFTOOL[uuid]:
+                if type(exif[key]) == list:
+                    assert sorted(exif[key]) == sorted(CLI_EXIFTOOL[uuid][key])
+                else:
                     assert exif[key] == CLI_EXIFTOOL[uuid][key]
 
 
@@ -1051,6 +1058,9 @@ def test_export_exiftool_ignore_date_modified():
                 CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid]["File:FileName"]
             ).asdict()
             for key in CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid]:
+                if type(exif[key]) == list:
+                    assert sorted(exif[key]) == sorted(CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid][key])
+                else:
                     assert exif[key] == CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid][key]
 
 
@@ -1093,6 +1103,38 @@ def test_export_exiftool_quicktime():
             for filename in files:
                 os.unlink(filename)
 
+
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_exiftool_duplicate_keywords():
+    """ ensure duplicate keywords are removed """
+    import glob
+    import os
+    import os.path
+    from osxphotos.__main__ import export
+    from osxphotos.exiftool import ExifTool
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        for uuid in CLI_EXIFTOOL_DUPLICATE_KEYWORDS:
+            result = runner.invoke(
+                export,
+                [
+                    os.path.join(cwd, PHOTOS_DB_15_7),
+                    ".",
+                    "-V",
+                    "--exiftool",
+                    "--uuid",
+                    f"{uuid}",
+                ],
+            )
+            exif = ExifTool(CLI_EXIFTOOL_DUPLICATE_KEYWORDS[uuid])
+            exifdict = exif.asdict()
+            assert sorted(exifdict["IPTC:Keywords"]) == ["Maria", "wedding"]
+            assert sorted(exifdict["XMP:Subject"]) == ["Maria", "wedding"]
+
+
 @pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
 def test_export_exiftool_error():
     """" test --exiftool catching error """
@@ -1106,7 +1148,7 @@ def test_export_exiftool_error():
     cwd = os.getcwd()
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
-        for uuid in CLI_EXIFTOOL_ERROR:
+        for uuid in CLI_EXIFTOOL:
             result = runner.invoke(
                 export,
                 [
@@ -1119,7 +1161,15 @@ def test_export_exiftool_error():
                 ],
             )
             assert result.exit_code == 0
-            assert "exiftool error" in result.output
+            files = glob.glob("*")
+            assert sorted(files) == sorted([CLI_EXIFTOOL[uuid]["File:FileName"]])
+
+            exif = ExifTool(CLI_EXIFTOOL[uuid]["File:FileName"]).asdict()
+            for key in CLI_EXIFTOOL[uuid]:
+                if type(exif[key]) == list:
+                    assert sorted(exif[key]) == sorted(CLI_EXIFTOOL[uuid][key])
+                else:
+                    assert exif[key] == CLI_EXIFTOOL[uuid][key]
 
 
 def test_export_edited_suffix():

tests/test_export_catalina_10_15_7.py

@@ -22,6 +22,7 @@ KEYWORDS = [
     "St. James's Park",
     "UK",
     "United Kingdom",
+    "Maria"
 ]
 # Photos 5 includes blank person for detected face
 PERSONS = ["Katie", "Suzy", "Maria", _UNKNOWN_PERSON]
@@ -39,6 +40,7 @@ KEYWORDS_DICT = {
     "St. James's Park": 1,
     "UK": 1,
     "United Kingdom": 1,
+    "Maria": 1,
 }
 PERSONS_DICT = {"Katie": 3, "Suzy": 2, "Maria": 1, _UNKNOWN_PERSON: 1}
 ALBUM_DICT = {
@@ -70,8 +72,8 @@ EXIF_JSON_UUID = UUID_DICT["has_adjustments"]
 EXIF_JSON_EXPECTED = """ 
     [{"EXIF:ImageDescription": "Bride Wedding day", 
     "XMP:Description": "Bride Wedding day", 
-    "XMP:TagsList": ["wedding"], 
+    "XMP:TagsList": ["Maria", "wedding"], 
-    "IPTC:Keywords": ["wedding"], 
+    "IPTC:Keywords": ["Maria", "wedding"], 
     "XMP:PersonInImage": ["Maria"], 
     "XMP:Subject": ["wedding", "Maria"], 
     "EXIF:DateTimeOriginal": "2019:04:15 14:40:24", 
@@ -85,8 +87,8 @@ EXIF_JSON_EXPECTED = """
 EXIF_JSON_EXPECTED_IGNORE_DATE_MODIFIED = """ 
     [{"EXIF:ImageDescription": "Bride Wedding day", 
     "XMP:Description": "Bride Wedding day", 
-    "XMP:TagsList": ["wedding"], 
+    "XMP:TagsList": ["Maria", "wedding"], 
-    "IPTC:Keywords": ["wedding"], 
+    "IPTC:Keywords": ["Maria", "wedding"], 
     "XMP:PersonInImage": ["Maria"], 
     "XMP:Subject": ["wedding", "Maria"], 
     "EXIF:DateTimeOriginal": "2019:04:15 14:40:24", 
@@ -522,8 +524,8 @@ def test_exiftool_json_sidecar_keyword_template_long(caplog):
         """
         [{"EXIF:ImageDescription": "Bride Wedding day", 
         "XMP:Description": "Bride Wedding day", 
-        "XMP:TagsList": ["wedding", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"], 
+        "XMP:TagsList": ["Maria", "wedding", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"], 
-        "IPTC:Keywords": ["wedding", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"], 
+        "IPTC:Keywords": ["Maria", "wedding", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"], 
         "XMP:PersonInImage": ["Maria"], 
         "XMP:Subject": ["wedding", "Maria"], 
         "EXIF:DateTimeOriginal": "2019:04:15 14:40:24", 
@@ -571,8 +573,8 @@ def test_exiftool_json_sidecar_keyword_template():
         """
         [{"EXIF:ImageDescription": "Bride Wedding day", 
         "XMP:Description": "Bride Wedding day", 
-        "XMP:TagsList": ["wedding", "Folder1/SubFolder2/AlbumInFolder", "I have a deleted twin"], 
+        "XMP:TagsList": ["Maria", "wedding", "Folder1/SubFolder2/AlbumInFolder", "I have a deleted twin"], 
-        "IPTC:Keywords": ["wedding", "Folder1/SubFolder2/AlbumInFolder", "I have a deleted twin"], 
+        "IPTC:Keywords": ["Maria", "wedding", "Folder1/SubFolder2/AlbumInFolder", "I have a deleted twin"], 
         "XMP:PersonInImage": ["Maria"], 
         "XMP:Subject": ["wedding", "Maria"],
         "EXIF:DateTimeOriginal": "2019:04:15 14:40:24", 

tests/test_export_mojave_10_14_6.py

@@ -427,9 +427,9 @@ def test_xmp_sidecar():
         <!-- keywords and persons listed in <dc:subject> as Photos does -->
         <dc:subject>
             <rdf:Seq>
+                <rdf:li>Katie</rdf:li>
                 <rdf:li>Kids</rdf:li>
                 <rdf:li>Suzy</rdf:li>
-                <rdf:li>Katie</rdf:li>
             </rdf:Seq>
         </dc:subject>
         <photoshop:DateCreated>2018-09-28T15:35:49.063000-04:00</photoshop:DateCreated>
@@ -438,8 +438,8 @@ def test_xmp_sidecar():
             xmlns:Iptc4xmpExt='http://iptc.org/std/Iptc4xmpExt/2008-02-29/'>
         <Iptc4xmpExt:PersonInImage>
             <rdf:Bag>
-                    <rdf:li>Suzy</rdf:li>
                     <rdf:li>Katie</rdf:li>
+                    <rdf:li>Suzy</rdf:li>
             </rdf:Bag>
         </Iptc4xmpExt:PersonInImage>
         </rdf:Description>