Fix for #515

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.42.80"
+__version__ = "0.42.81"

osxphotos/photoinfo/_photoinfo_export.py

@@ -56,7 +56,7 @@ from ..photokit import (
 )
 from ..phototemplate import RenderOptions
 from ..uti import get_preferred_uti_extension
-from ..utils import findfiles, lineno, noop
+from ..utils import findfiles, lineno, noop, normalize_fs_path
 
 # retry if use_photos_export fails the first time (which sometimes it does)
 MAX_PHOTOSCRIPT_RETRIES = 3
@@ -686,9 +686,12 @@ def export2(
     count = 0
     if not update and increment and not overwrite:
         dest_files = findfiles(f"{dest_original.stem}*", str(dest_original.parent))
-        dest_files = [pathlib.Path(f).stem.lower() for f in dest_files]
+        # paths need to be normalized for unicode as filesystem returns unicode in NFD form
+        dest_files = [
+            normalize_fs_path(pathlib.Path(f).stem.lower()) for f in dest_files
+        ]
         dest_new = dest_original.stem
-        while dest_new.lower() in dest_files:
+        while normalize_fs_path(dest_new.lower()) in dest_files:
             count += 1
             dest_new = f"{dest_original.stem} ({count})"
         dest_original = dest_original.parent / f"{dest_new}{dest_original.suffix}"
@@ -708,12 +711,15 @@ def export2(
     if export_edited:
         if not update and increment and not overwrite:
             dest_files = findfiles(f"{dest_edited.stem}*", str(dest_edited.parent))
-            dest_files = [pathlib.Path(f).stem.lower() for f in dest_files]
+            # paths need to be normalized for unicode as filesystem returns unicode in NFD form
+            dest_files = [
+                normalize_fs_path(pathlib.Path(f).stem.lower()) for f in dest_files
+            ]
             dest_new = dest_edited.stem
             if count:
                 # incremented above when checking original destination
                 dest_new = f"{dest_new} ({count})"
-            while dest_new.lower() in dest_files:
+            while normalize_fs_path(dest_new.lower()) in dest_files:
                 count += 1
                 dest_new = f"{dest.stem} ({count})"
             dest_edited = dest_edited.parent / f"{dest_new}{dest_edited.suffix}"

osxphotos/utils.py

@@ -19,6 +19,8 @@ from plistlib import load as plistload
 from typing import Callable
 
 import CoreFoundation
+import objc
+from Foundation import NSString
 
 from ._constants import UNICODE_FORMAT
 
@@ -263,6 +265,13 @@ def list_photo_libraries():
     return lib_list
 
 
+def normalize_fs_path(path: str) -> str:
+    """Normalize filesystem paths with unicode in them"""
+    with objc.autorelease_pool():
+        normalized_path = NSString.fileSystemRepresentation(path)
+        return normalized_path.decode('utf8')
+
+
 def findfiles(pattern, path_):
     """Returns list of filenames from path_ matched by pattern
     shell pattern. Matching is case-insensitive.
@@ -271,8 +280,11 @@ def findfiles(pattern, path_):
         return []
     # See: https://gist.github.com/techtonik/5694830
 
+    # paths need to be normalized for unicode as filesystem returns unicode in NFD form
+    pattern = normalize_fs_path(pattern)
     rule = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
-    return [name for name in os.listdir(path_) if rule.match(name)]
+    files = [normalize_fs_path(p) for p in os.listdir(path_)]
+    return [name for name in files if rule.match(name)]
 
 
 def _open_sql_file(dbname):

requirements.txt

@@ -1,23 +1,23 @@
-pyobjc-core>=7.2
+pyobjc-core>=7.2,<8.0
-pyobjc-framework-AppleScriptKit>=7.2
+pyobjc-framework-AppleScriptKit>=7.2,<8.0
-pyobjc-framework-AppleScriptObjC>=7.2
+pyobjc-framework-AppleScriptObjC>=7.2,<8.0
-pyobjc-framework-Photos>=7.2
+pyobjc-framework-Photos>=7.2,<8.0
-pyobjc-framework-Quartz>=7.2
+pyobjc-framework-Quartz>=7.2,<8.0
-pyobjc-framework-AVFoundation>=7.2
+pyobjc-framework-AVFoundation>=7.2,<8.0
-pyobjc-framework-CoreServices>=7.2
+pyobjc-framework-CoreServices>=7.2,<8.0
-pyobjc-framework-Metal>=7.2
+pyobjc-framework-Metal>=7.2,<8.0
-pyobjc-framework-Vision>=7.2
+pyobjc-framework-Vision>=7.2,<8.0
-Click==8.0.1
+Click>=8.0.1,<9.0
-PyYAML==5.4.1
+PyYAML>=5.4.1<5.5.0
-Mako==1.1.4
+Mako>=1.1.4,<1.2.0
 bpylist2==3.0.2
-pathvalidate==2.4.1
+pathvalidate>=2.4.1,<2.5.0
 dataclasses==0.7;python_version<'3.7'
-wurlitzer==2.1.0
+wurlitzer>=2.1.0,<2.2.0
-photoscript==0.1.4
+photoscript>=0.1.4,<0.2.0
-toml==0.10.2
+toml>=0.10.2,<0.11.0
-osxmetadata==0.99.33
+osxmetadata>=0.99.33,<1.0.0
-textx==2.3.0
+textx>=2.3.0,<2.4.0
-rich==10.6.0
+rich>=10.6.0,<11.0.0
-bitmath==1.3.3.1
+bitmath>=1.3.3.1,<1.4.0.0
-more-itertools==8.8.0 
+more-itertools>=8.8.0,<9.0.0

setup.py

@@ -73,29 +73,29 @@ setup(
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
     install_requires=[
-        "pyobjc-core",
+        "pyobjc-core>=7.2,<8.0",
-        "pyobjc-framework-AppleScriptKit",
+        "pyobjc-framework-AppleScriptKit>=7.2,<8.0",
-        "pyobjc-framework-AppleScriptObjC",
+        "pyobjc-framework-AppleScriptObjC>=7.2,<8.0",
-        "pyobjc-framework-Photos",
+        "pyobjc-framework-Photos>=7.2,<8.0",
-        "pyobjc-framework-Quartz",
+        "pyobjc-framework-Quartz>=7.2,<8.0",
-        "pyobjc-framework-AVFoundation",
+        "pyobjc-framework-AVFoundation>=7.2,<8.0",
-        "pyobjc-framework-CoreServices",
+        "pyobjc-framework-CoreServices>=7.2,<8.0",
-        "pyobjc-framework-Metal",
+        "pyobjc-framework-Metal>=7.2,<8.0",
-        "pyobjc-framework-Vision",
+        "pyobjc-framework-Vision>=7.2,<8.0",
-        "Click==8.0.1",
+        "Click>=8.0.1,<9.0",
-        "PyYAML==5.4.1",
+        "PyYAML>=5.4.1,<5.5.0",
-        "Mako==1.1.4",
+        "Mako>=1.1.4,<1.2.0",
         "bpylist2==3.0.2",
-        "pathvalidate==2.4.1",
+        "pathvalidate>=2.4.1,<2.5.0",
         "dataclasses==0.7;python_version<'3.7'",
-        "wurlitzer==2.1.0",
+        "wurlitzer>=2.1.0,<2.2.0",
-        "photoscript==0.1.4",
+        "photoscript>=0.1.4,<0.2.0",
-        "toml==0.10.2",
+        "toml>=0.10.2,<0.11.0",
-        "osxmetadata==0.99.33",
+        "osxmetadata>=0.99.33,<1.0.0",
-        "textx==2.3.0",
+        "textx>=2.3.0,<2.4.0",
-        "rich==10.6.0",
+        "rich>=10.6.0,<11.0.0",
-        "bitmath==1.3.3.1",
+        "bitmath>=1.3.3.1,<1.4.0.0",
-        "more-itertools==8.8.0",
+        "more-itertools>=8.8.0,<9.0.0",
     ],
     entry_points={"console_scripts": ["osxphotos=osxphotos.__main__:cli"]},
     include_package_data=True,

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2021-07-20T05:48:01Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2021-07-20T05:48:00Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2021-07-20T07:05:31Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2021-07-20T05:48:01Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2021-07-20T05:48:00Z</date>
+	<date>2021-09-14T04:40:41Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2021-07-20T05:48:01Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2021-07-20T05:48:08Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2021-07-20T05:47:59Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2021-07-20T05:48:01Z</date>
+	<date>2021-09-14T04:40:43Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2021-07-20T05:48:01Z</date>
+	<date>2021-09-14T04:40:42Z</date>
 </dict>
 </plist>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/vision/PhotoAnalysisServicePreferences.plist

@@ -3,8 +3,8 @@
 <plist version="1.0">
 <dict>
 	<key>FaceIDModelLastGenerationKey</key>
-	<date>2021-07-20T05:48:02Z</date>
+	<date>2021-09-14T04:49:52Z</date>
 	<key>LastContactClassificationKey</key>
-	<date>2021-07-20T05:48:05Z</date>
+	<date>2021-09-14T04:51:05Z</date>
 </dict>
 </plist>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/vision/clustererState.plist

@@ -3,6 +3,6 @@
 <plist version="1.0">
 <dict>
 	<key>PVClustererBringUpState</key>
-	<integer>50</integer>
+	<integer>40</integer>
 </dict>
 </plist>

tests/Test-10.15.7.photoslibrary/resources/renders/1/1793FAAB-DE75-4E25-886C-2BD66C780D6A.plist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>adjustmentBaseVersion</key>
+	<integer>0</integer>
+	<key>adjustmentData</key>
+	<data>
+	bZHNTsMwEITfZc8hcn4aaG5wabkUiSKKhDhs602zEDuRvemlyrtjt2pBiKN3v5mdkY9w
+	IOe5t4+26aE+wnbkTq9GsyUHNWTzZTaDBHAYXs9cHFZZqtIsV2Hhdy0ZfKYDn5dZAkOH
+	0vTOBPJp/QZTAoYENQpGf4NeyG1YSwt1qYo8CHigji39XAi6tAzuZ3hJvG8F6kLlZQK9
+	Y7KCciKr4B5voVzFIQHqz9GLCZiH+v34D0EWtx1pqMWNFFqQCNu9jwHZDqM8dLj7urd6
+	07IQ1DcqVYUqqlKVVTHPbmd5pe5ClKYJyoVDDq7q8l6LI7uP9a6jFY3isFugMXga+1jA
+	C+/iyemCLUf6JXrpbXyGLetQhRs+fcnaoPuTb/qYvgE=
+	</data>
+	<key>adjustmentEditorBundleID</key>
+	<string>com.apple.Photos</string>
+	<key>adjustmentFormatIdentifier</key>
+	<string>com.apple.photo</string>
+	<key>adjustmentFormatVersion</key>
+	<string>1.4</string>
+	<key>adjustmentTimestamp</key>
+	<date>2021-09-14T04:49:50Z</date>
+</dict>
+</plist>

tests/Test-10.15.7.photoslibrary/resources/renders/D/D1D4040D-D141-44E8-93EA-E403D9F63E07.plist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>adjustmentBaseVersion</key>
+	<integer>0</integer>
+	<key>adjustmentData</key>
+	<data>
+	bZFNb8IwDIb/S86oSgp0o7dxgV2YNKYxadrBEJd6a9Iqcbmg/vc5VLBp2jH2835YOasT
+	hkitf/RVq8qz2vfU2E3v9hhUqcxibeZqoqDrXkcuDQuT6czkWhbxUKODZzzRuDQT1TXA
+	VRuckE/bNzVMlEMGCwzJ30FkDDuyXAut77UIqMOGPP4kiC6bifsIr5GONauy0OLeBkLP
+	wGOamKco4JtWELCffWQnWFTl+/kfAj3sG7Sq5NCjHIHM5I8x9SPf9bxs4PD14O2uJkZV
+	6qyY52aWTxfFQt/lpphNpUhViW4VgMRTX99bDuiP6bbbaIM9B2hW4BxcxjHVj0yHFDhc
+	sXWPv0QvrU9P2ZKVQ6iiy39sHYQ/7YaP4Rs=
+	</data>
+	<key>adjustmentEditorBundleID</key>
+	<string>com.apple.Photos</string>
+	<key>adjustmentFormatIdentifier</key>
+	<string>com.apple.photo</string>
+	<key>adjustmentFormatVersion</key>
+	<string>1.4</string>
+	<key>adjustmentTimestamp</key>
+	<date>2021-09-14T04:50:39Z</date>
+</dict>
+</plist>

tests/test_catalina_10_15_7.py

@@ -23,10 +23,10 @@ PHOTOS_DB = "tests/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_DB_PATH = "/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_LIBRARY_PATH = "/Test-10.15.7.photoslibrary"
 
-PHOTOS_DB_LEN = 21
+PHOTOS_DB_LEN = 25
-PHOTOS_NOT_IN_TRASH_LEN = 19
+PHOTOS_NOT_IN_TRASH_LEN = 23
 PHOTOS_IN_TRASH_LEN = 2
-PHOTOS_DB_IMPORT_SESSIONS = 15
+PHOTOS_DB_IMPORT_SESSIONS = 17
 
 KEYWORDS = [
     "Kids",
@@ -45,6 +45,15 @@ KEYWORDS = [
     "Val d'Isère",
     "Wine",
     "Wine Bottle",
+    "Food",
+    "Furniture",
+    "Pizza",
+    "Table",
+    "Cloudy",
+    "Cord",
+    "Outdoor",
+    "Sky",
+    "Sunset Sunrise",
 ]
 # Photos 5 includes blank person for detected face
 PERSONS = ["Katie", "Suzy", "Maria", _UNKNOWN_PERSON]
@@ -80,6 +89,15 @@ KEYWORDS_DICT = {
     "flowers": 1,
     "foo/bar": 1,
     "wedding": 3,
+    "Food": 2,
+    "Furniture": 2,
+    "Pizza": 2,
+    "Table": 2,
+    "Cloudy": 2,
+    "Cord": 2,
+    "Outdoor": 2,
+    "Sky": 2,
+    "Sunset Sunrise": 2,
 }
 PERSONS_DICT = {"Katie": 3, "Suzy": 2, "Maria": 2, _UNKNOWN_PERSON: 1}
 ALBUM_DICT = {
@@ -165,6 +183,12 @@ UTI_ORIGINAL_DICT = {
     "1EB2B765-0765-43BA-A90C-0D0580E6172C": "public.jpeg",
 }
 
+UUID_UNICODE_TITLE = [
+    "B13F4485-94E0-41CD-AF71-913095D62E31", # Frítest.jpg
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A", # Frítest.jpg
+    "A8266C97-9BAF-4AF4-99F3-0013832869B8", # Frítest.jpg
+    "D1D4040D-D141-44E8-93EA-E403D9F63E07", # Frítest.jpg
+]
 
 RawInfo = namedtuple(
     "RawInfo",
@@ -1073,7 +1097,7 @@ def test_from_to_date(photosdb):
     time.tzset()
 
     photos = photosdb.photos(from_date=datetime.datetime(2018, 10, 28))
-    assert len(photos) == 12
+    assert len(photos) == 16
 
     photos = photosdb.photos(to_date=datetime.datetime(2018, 10, 28))
     assert len(photos) == 7

tests/test_cli.py

@@ -800,6 +800,22 @@ DESCRIPTION_VALUE_EMPTY_TITLE =  "No Title and No Descr"
 DESCRIPTION_TEMPLATE_TITLE_CONDITIONAL = "{title?true,false}"
 DESCRIPTION_VALUE_TITLE_CONDITIONAL = "false"
 
+
+UUID_UNICODE_TITLE = [
+    "B13F4485-94E0-41CD-AF71-913095D62E31",  # Frítest.jpg
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A",  # Frítest.jpg
+    "A8266C97-9BAF-4AF4-99F3-0013832869B8",  # Frítest.jpg
+    "D1D4040D-D141-44E8-93EA-E403D9F63E07",  # Frítest.jpg
+]
+
+EXPORT_UNICODE_TITLE_FILENAMES = [
+    "Frítest.jpg",
+    "Frítest (1).jpg",
+    "Frítest (2).jpg",
+    "Frítest (3).jpg",
+]
+
+
 def modify_file(filename):
     """appends data to a file to modify it"""
     with open(filename, "ab") as fd:
@@ -2152,6 +2168,51 @@ def test_export_duplicate():
         assert len(files) == len(UUID_DUPLICATES)
 
 
+def test_export_duplicate_unicode_filenames():
+    # test issue #515
+    import glob
+    import os
+    import os.path
+
+    import osxphotos
+    from osxphotos.cli import export
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    uuid = []
+    for u in UUID_UNICODE_TITLE:
+        uuid.append("--uuid")
+        uuid.append(u)
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--convert-to-jpeg",
+                "--edited-suffix",
+                "",
+                "--filename",
+                "{title,{original_name}}",
+                "--jpeg-ext",
+                "jpg",
+                "--person-keyword",
+                "--skip-bursts",
+                "--skip-live",
+                "--skip-original-if-edited",
+                "--touch-file",
+                "--strip",
+                *uuid,
+                "-V",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "exported: 4" in result.output
+        files = glob.glob("*")
+        assert sorted(files) == sorted(EXPORT_UNICODE_TITLE_FILENAMES)
+
+
 def test_query_date_1():
     """Test --from-date and --to-date"""
     import json
@@ -7156,13 +7217,14 @@ def test_export_description_template():
                 "-V",
                 "--description-template",
                 DESCRIPTION_TEMPLATE_EMPTY_TITLE,
-                "--exiftool"
+                "--exiftool",
             ],
         )
         assert result.exit_code == 0
         exif = ExifTool(FILENAME_EMPTY_TITLE).asdict()
         assert exif["EXIF:ImageDescription"] == DESCRIPTION_VALUE_EMPTY_TITLE
 
+
 def test_export_description_template_conditional():
     """Test for issue #506"""
     import json
@@ -7191,12 +7253,12 @@ def test_export_description_template_conditional():
                 "--description-template",
                 DESCRIPTION_TEMPLATE_TITLE_CONDITIONAL,
                 "--sidecar",
-                "JSON"
+                "JSON",
-            
             ],
         )
         assert result.exit_code == 0
         with open(f"{FILENAME_EMPTY_TITLE}.json", "r") as fp:
             json_got = json.load(fp)[0]
-            assert json_got["EXIF:ImageDescription"] == DESCRIPTION_VALUE_TITLE_CONDITIONAL
+            assert (
-
+                json_got["EXIF:ImageDescription"] == DESCRIPTION_VALUE_TITLE_CONDITIONAL
+            )