Fix for #561, no really, I mean it this time

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.45.4"
+__version__ = "0.45.5"

osxphotos/export_db.py

@@ -10,13 +10,17 @@ import sys
 from abc import ABC, abstractmethod
 from io import StringIO
 from sqlite3 import Error
+from typing import Union
 
 from ._constants import OSXPHOTOS_EXPORT_DB
 from ._version import __version__
+from .utils import normalize_fs_path
 
 __all__ = ["ExportDB_ABC", "ExportDBNoOp", "ExportDB", "ExportDBInMemory"]
 
-OSXPHOTOS_EXPORTDB_VERSION = "4.2"
+OSXPHOTOS_EXPORTDB_VERSION = "4.3"
+OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH = "4.3"
+
 OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
 
 
@@ -211,12 +215,13 @@ class ExportDB(ExportDB_ABC):
         """query database for filename and return UUID
         returns None if filename not found in database
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filepath_normalized = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
             c.execute(
-                "SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,)
+                "SELECT uuid FROM files WHERE filepath_normalized = ?",
+                (filepath_normalized,),
             )
             results = c.fetchone()
             uuid = results[0] if results else None
@@ -228,7 +233,7 @@ class ExportDB(ExportDB_ABC):
     def set_uuid_for_file(self, filename, uuid):
         """set UUID of filename to uuid in the database"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -245,7 +250,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -266,7 +271,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -302,7 +307,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename (after exiftool has updated it)
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -323,7 +328,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename (after exiftool has updated it)
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -384,7 +389,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_exifdata_for_file(self, filename):
         """returns the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -402,7 +407,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_exifdata_for_file(self, filename, exifdata):
         """sets the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -416,7 +421,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_sidecar_for_file(self, filename):
         """returns the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -444,7 +449,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
         """sets the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -515,7 +520,7 @@ class ExportDB(ExportDB_ABC):
     ):
         """sets all the data for file and uuid at once; if any value is None, does not set it"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -577,7 +582,7 @@ class ExportDB(ExportDB_ABC):
             logging.warning(e)
 
     def _set_stat_for_file(self, table, filename, stats):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -590,7 +595,7 @@ class ExportDB(ExportDB_ABC):
         conn.commit()
 
     def _get_stat_for_file(self, table, filename):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         c = conn.cursor()
         c.execute(
@@ -626,6 +631,8 @@ class ExportDB(ExportDB_ABC):
             version_info = self._get_database_version(conn)
             if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION:
                 self._create_db_tables(conn)
+                if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH:
+                    self._migrate_normalized_filepath(conn)
                 self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
             else:
                 self.was_upgraded = ()
@@ -782,6 +789,32 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+    def _normalize_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, lower case"""
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _normalize_filepath_relative(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, relative path (to export dir), lower case"""
+        filepath = str(pathlib.Path(filepath).relative_to(self._path))
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _migrate_normalized_filepath(self, conn):
+        """Fix all filepath_normalized columns for unicode normalization"""
+        # Prior to database version 4.3, filepath_normalized was not normalized for unicode
+        c = conn.cursor()
+        for table in ["converted", "edited", "exifdata", "files", "sidecar"]:
+            old_values = c.execute(
+                f"SELECT filepath_normalized, id FROM {table}"
+            ).fetchall()
+            new_values = [
+                (self._normalize_filepath(filepath_normalized), id_)
+                for filepath_normalized, id_ in old_values
+            ]
+            c.executemany(
+                f"UPDATE {table} SET filepath_normalized=? WHERE id=?", new_values
+            )
+        conn.commit()
+
 
 class ExportDBInMemory(ExportDB):
     """In memory version of ExportDB

tests/Test-10.15.7.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>585B80BF-8D1F-55EF-A9E8-6CF4E5523959</string>
 	<key>pid</key>
-	<integer>1961</integer>
+	<integer>14817</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2021-09-14T04:40:41Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2021-07-20T05:48:08Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2021-07-20T05:47:59Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2021-09-14T04:40:43Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 </dict>
 </plist>

tests/test_albums_folders_catalina_10_15_7.py

@@ -21,6 +21,7 @@ FOLDER_ALBUM_DICT = {
 ALBUM_NAMES = [
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum",
     "2019-10/11 Paris Clermont",
+    "Água",
     "AlbumInFolder",
     "EmptyAlbum",
     "I have a deleted twin",
@@ -38,6 +39,7 @@ ALBUM_NAMES = [
 ALBUM_PARENT_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": None,
     "2019-10/11 Paris Clermont": None,
+    "Água": None,
     "AlbumInFolder": "SubFolder2",
     "EmptyAlbum": None,
     "I have a deleted twin": None,
@@ -54,6 +56,7 @@ ALBUM_PARENT_DICT = {
 ALBUM_FOLDER_NAMES_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": [],
     "2019-10/11 Paris Clermont": [],
+    "Água": [],
     "AlbumInFolder": ["Folder1", "SubFolder2"],
     "EmptyAlbum": [],
     "I have a deleted twin": [],
@@ -70,6 +73,7 @@ ALBUM_FOLDER_NAMES_DICT = {
 ALBUM_LEN_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": 1,
     "2019-10/11 Paris Clermont": 1,
+    "Água": 3,
     "AlbumInFolder": 2,
     "EmptyAlbum": 0,
     "I have a deleted twin": 1,
@@ -103,6 +107,11 @@ ALBUM_PHOTO_UUID_DICT = {
         "4D521201-92AC-43E5-8F7C-59BC41C37A96",
         "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
     ],
+    "Água": [
+        "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091",
+        "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
+        "54E76FCB-D353-4557-9997-0A457BCB4D48",
+    ],
 }
 
 UUID_DICT = {

tests/test_catalina_10_15_7.py

@@ -24,10 +24,10 @@ PHOTOS_DB = "tests/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_DB_PATH = "/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_LIBRARY_PATH = "/Test-10.15.7.photoslibrary"
 
-PHOTOS_DB_LEN = 25
+PHOTOS_DB_LEN = 29
-PHOTOS_NOT_IN_TRASH_LEN = 23
+PHOTOS_NOT_IN_TRASH_LEN = 27
 PHOTOS_IN_TRASH_LEN = 2
-PHOTOS_DB_IMPORT_SESSIONS = 17
+PHOTOS_DB_IMPORT_SESSIONS = 21
 
 KEYWORDS = [
     "Kids",
@@ -72,6 +72,7 @@ ALBUMS = [
     "Sorted Oldest First",
     "Sorted Title",
     "Test Album",  # there are 2 albums named "Test Album" for testing duplicate album names
+    "Água",
 ]
 KEYWORDS_DICT = {
     "Drink": 2,
@@ -115,6 +116,7 @@ ALBUM_DICT = {
     "Sorted Oldest First": 3,
     "Sorted Title": 3,
     "Test Album": 2,
+    "Água": 3,
 }  # Note: there are 2 albums named "Test Album" for testing duplicate album names
 
 UUID_DICT = {
@@ -1091,7 +1093,7 @@ def test_from_to_date(photosdb):
     time.tzset()
 
     photos = photosdb.photos(from_date=datetime.datetime(2018, 10, 28))
-    assert len(photos) == 16
+    assert len(photos) == 20
 
     photos = photosdb.photos(to_date=datetime.datetime(2018, 10, 28))
     assert len(photos) == 7

tests/test_cli.py

@@ -79,64 +79,69 @@ CLI_OUTPUT_NO_SUBCOMMAND = [
 CLI_OUTPUT_QUERY_UUID = '[{"uuid": "D79B8D77-BFFC-460B-9312-034F2877D35B", "filename": "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg", "original_filename": "Pumkins2.jpg", "date": "2018-09-28T16:07:07-04:00", "description": "Girl holding pumpkin", "title": "I found one!", "keywords": ["Kids"], "albums": ["Pumpkin Farm", "Test Album", "Multi Keyword"], "persons": ["Katie"], "path": "/tests/Test-10.15.7.photoslibrary/originals/D/D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 41.256566, "longitude": -95.940257, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null}]'
 
 CLI_EXPORT_FILENAMES = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 
 CLI_EXPORT_FILENAMES_DRY_RUN = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
     "IMG_1693.tif",
-    "IMG_1994.JPG",
     "IMG_1994.cr2",
-    "IMG_1997.JPG",
+    "IMG_1994.JPG",
     "IMG_1997.cr2",
-    "IMG_3092.heic",
+    "IMG_1997.JPG",
     "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
     "IMG_4547.jpg",
     "Jellyfish.MOV",
     "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
     "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
     "winebottle.jpeg",
     "winebottle.jpeg",
-    "Frítest.jpg",
-    "Frítest_edited.jpeg",
 ]
 
 CLI_EXPORT_IGNORE_SIGNATURE_FILENAMES = ["Tulips.jpg", "wedding.jpg"]
@@ -154,225 +159,253 @@ CLI_EXPORT_ORIGINAL_SUFFIX_TEMPLATE = "{edited?_original,}"
 CLI_EXPORT_PREVIEW_SUFFIX = "_lowres"
 
 CLI_EXPORT_FILENAMES_EDITED_SUFFIX = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_bearbeiten.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_bearbeiten.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_bearbeiten.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_bearbeiten.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_bearbeiten.jpeg",
     "Frítest_bearbeiten (1).jpeg",
+    "Frítest_bearbeiten.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_bearbeiten.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_bearbeiten.jpeg",
+    "St James Park.jpg",
+    "Tulips_bearbeiten.jpeg",
+    "Tulips.jpg",
+    "wedding_bearbeiten.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_EDITED_SUFFIX_TEMPLATE = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_ORIGINAL_SUFFIX = [
-    "Pumkins1_original.jpg",
+    "[2020-08-29] AAF035_original (1).jpg",
-    "Pumkins2_original.jpg",
+    "[2020-08-29] AAF035_original (2).jpg",
-    "Pumpkins3_original.jpg",
+    "[2020-08-29] AAF035_original (3).jpg",
-    "St James Park_original.jpg",
+    "[2020-08-29] AAF035_original.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips_original.jpg",
-    "wedding_original.jpg",
-    "wedding_edited.jpeg",
     "DSC03584_original.dng",
-    "IMG_1693_original.tif",
+    "Frítest_edited (1).jpeg",
-    "IMG_1994_original.JPG",
+    "Frítest_edited.jpeg",
-    "IMG_1994_original.cr2",
-    "IMG_1997_original.JPG",
-    "IMG_1997_original.cr2",
-    "IMG_3092_original.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547_original.jpg",
-    "Jellyfish_original.MOV",
-    "Jellyfish1_original.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png_original.jpeg",
-    "winebottle_original.jpeg",
-    "winebottle_original (1).jpeg",
-    "Frítest_original.jpg",
     "Frítest_original (1).jpg",
     "Frítest_original (2).jpg",
     "Frítest_original (3).jpg",
-    "Frítest_edited.jpeg",
+    "Frítest_original.jpg",
-    "Frítest_edited (1).jpeg",
+    "IMG_1693_original.tif",
+    "IMG_1994_original.cr2",
+    "IMG_1994_original.JPG",
+    "IMG_1997_original.cr2",
+    "IMG_1997_original.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092_original.heic",
+    "IMG_4547_original.jpg",
+    "Jellyfish_original.MOV",
+    "Jellyfish1_original.mp4",
+    "Pumkins1_original.jpg",
+    "Pumkins2_original.jpg",
+    "Pumpkins3_original.jpg",
+    "screenshot-really-a-png_original.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park_original.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips_original.jpg",
+    "wedding_edited.jpeg",
+    "wedding_original.jpg",
+    "winebottle_original (1).jpeg",
+    "winebottle_original.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_ORIGINAL_SUFFIX_TEMPLATE = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park_original.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips_original.jpg",
-    "wedding_original.jpg",
-    "wedding_edited.jpeg",
-    "Tulips_edited.jpeg",
     "DSC03584.dng",
+    "Frítest (1).jpg",
+    "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest_original (1).jpg",
+    "Frítest_original.jpg",
+    "Frítest.jpg",
     "IMG_1693.tif",
-    "IMG_1994.JPG",
     "IMG_1994.cr2",
-    "IMG_1997.JPG",
+    "IMG_1994.JPG",
     "IMG_1997.cr2",
-    "IMG_3092_original.heic",
+    "IMG_1997.JPG",
     "IMG_3092_edited.jpeg",
+    "IMG_3092_original.heic",
     "IMG_4547.jpg",
     "Jellyfish.MOV",
     "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
     "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park_original.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips_original.jpg",
+    "wedding_edited.jpeg",
+    "wedding_original.jpg",
     "winebottle (1).jpeg",
-    "Frítest.jpg",
+    "winebottle.jpeg",
-    "Frítest (1).jpg",
-    "Frítest_original.jpg",
-    "Frítest_edited.jpeg",
-    "Frítest_original (1).jpg",
-    "Frítest_edited (1).jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CURRENT = [
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A_edited.jpeg",  # Frítest.jpg
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A.jpeg",  # Frítest.jpg
     "1EB2B765-0765-43BA-A90C-0D0580E6172C.jpeg",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A.jpeg",
+    "35329C57-B963-48D6-BB75-6AFF9370CBBC.mov",
     "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg",
     "4D521201-92AC-43E5-8F7C-59BC41C37A96.cr2",
     "4D521201-92AC-43E5-8F7C-59BC41C37A96.jpeg",
+    "52083079-73D5-4921-AC1B-FE76F279133F.jpeg",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48.jpeg",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4_edited.jpeg",
     "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4.jpeg",
+    "7783E8E6-9CAC-40F3-BE22-81FB7051C266_edited.jpeg",
+    "7783E8E6-9CAC-40F3-BE22-81FB7051C266.heic",
+    "7F74DD34-5920-4DA3-B284-479887A34F66.jpeg",
+    "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091.jpeg",
+    "8846E3E6-8AC8-4857-8448-E3D025784410.tiff",
+    "A8266C97-9BAF-4AF4-99F3-0013832869B8.jpeg",  # Frítest.jpg
     "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.cr2",
     "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.jpeg",
-    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068.dng",
-    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30.jpeg",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30_edited.jpeg",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51_edited.jpeg",
-    "F12384F6-CD17-4151-ACBA-AE0E3688539E.jpeg",
-    "35329C57-B963-48D6-BB75-6AFF9370CBBC.mov",
-    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4_edited.jpeg",
-    "7783E8E6-9CAC-40F3-BE22-81FB7051C266.heic",
-    "7783E8E6-9CAC-40F3-BE22-81FB7051C266_edited.jpeg",
-    "7F74DD34-5920-4DA3-B284-479887A34F66.jpeg",
-    "8846E3E6-8AC8-4857-8448-E3D025784410.tiff",
-    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3.mp4",
-    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91.jpeg",
-    "52083079-73D5-4921-AC1B-FE76F279133F.jpeg",
     "B13F4485-94E0-41CD-AF71-913095D62E31.jpeg",  # Frítest.jpg
-    "1793FAAB-DE75-4E25-886C-2BD66C780D6A.jpeg",  # Frítest.jpg
+    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068.dng",
-    "1793FAAB-DE75-4E25-886C-2BD66C780D6A_edited.jpeg",  # Frítest.jpg
+    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3.mp4",
-    "A8266C97-9BAF-4AF4-99F3-0013832869B8.jpeg",  # Frítest.jpg
-    "D1D4040D-D141-44E8-93EA-E403D9F63E07.jpeg",  # Frítest.jpg
     "D1D4040D-D141-44E8-93EA-E403D9F63E07_edited.jpeg",  # Frítest.jpg
+    "D1D4040D-D141-44E8-93EA-E403D9F63E07.jpeg",  # Frítest.jpg
+    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
+    "DC99FBDD-7A52-4100-A5BB-344131646C30_edited.jpeg",
+    "DC99FBDD-7A52-4100-A5BB-344131646C30.jpeg",
+    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91.jpeg",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51_edited.jpeg",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg",
+    "F12384F6-CD17-4151-ACBA-AE0E3688539E.jpeg",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CONVERT_TO_JPEG = [
+    "[2020-08-29] AAF035 (1).jpg",
+    "[2020-08-29] AAF035 (2).jpg",
+    "[2020-08-29] AAF035 (3).jpg",
+    "[2020-08-29] AAF035.jpg",
     "DSC03584.jpeg",
-    "IMG_1693.jpeg",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.jpeg",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Pumkins1.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "Tulips_edited.jpeg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
     "Frítest_edited (1).jpeg",
     "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.jpeg",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.jpeg",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CONVERT_TO_JPEG_SKIP_RAW = [
+    "[2020-08-29] AAF035 (1).jpg",
+    "[2020-08-29] AAF035 (2).jpg",
+    "[2020-08-29] AAF035 (3).jpg",
+    "[2020-08-29] AAF035.jpg",
     "DSC03584.jpeg",
-    "IMG_1693.jpeg",
-    "IMG_1994.JPG",
-    "IMG_1997.JPG",
-    "IMG_3092.jpeg",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Pumkins1.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "Tulips_edited.jpeg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.jpeg",
+    "IMG_1994.JPG",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.jpeg",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_CONVERT_TO_JPEG_LARGE_FILE = "DSC03584.jpeg"
@@ -546,7 +579,7 @@ PHOTOS_NOT_IN_TRASH_LEN_14_6 = 12
 PHOTOS_IN_TRASH_LEN_14_6 = 1
 PHOTOS_MISSING_14_6 = 1
 
-PHOTOS_NOT_IN_TRASH_LEN_15_7 = 23
+PHOTOS_NOT_IN_TRASH_LEN_15_7 = 27
 PHOTOS_IN_TRASH_LEN_15_7 = 2
 PHOTOS_MISSING_15_7 = 2
 PHOTOS_EDITED_15_7 = 6
@@ -732,6 +765,7 @@ ALBUMS_JSON = {
         "Sorted Newest First": 3,
         "Sorted Oldest First": 3,
         "Sorted Title": 3,
+        "Água": 3,
     },
     "shared albums": {},
 }
@@ -746,6 +780,7 @@ ALBUMS_STR = """albums:
   2018-10 - Sponsion, Museum, Frühstück, Römermuseum: 1
   2019-10/11 Paris Clermont: 1
   EmptyAlbum: 0
+  Água: 3
 shared albums: {}
 """
 
@@ -820,37 +855,45 @@ UUID_IS_REFERENCE = [
 ]
 
 UUID_IN_ALBUM = [
-    "F12384F6-CD17-4151-ACBA-AE0E3688539E",
-    "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
     "1EB2B765-0765-43BA-A90C-0D0580E6172C",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
-    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
-    "D79B8D77-BFFC-460B-9312-034F2877D35B",
-    "4D521201-92AC-43E5-8F7C-59BC41C37A96",
-    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068",
     "3DD2C897-F19E-4CA6-8C22-B027D5A71907",
+    "4D521201-92AC-43E5-8F7C-59BC41C37A96",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48",
     "7783E8E6-9CAC-40F3-BE22-81FB7051C266",
+    "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091",
+    "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
+    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068",
+    "D79B8D77-BFFC-460B-9312-034F2877D35B",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51",
+    "F12384F6-CD17-4151-ACBA-AE0E3688539E",
 ]
 
 UUID_NOT_IN_ALBUM = [
-    "A1DD1F98-2ECD-431F-9AC9-5AFEFE2D3A5C",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30",
-    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3",
-    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91",
-    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
-    "35329C57-B963-48D6-BB75-6AFF9370CBBC",
-    "8846E3E6-8AC8-4857-8448-E3D025784410",
-    "7F74DD34-5920-4DA3-B284-479887A34F66",
-    "52083079-73D5-4921-AC1B-FE76F279133F",
-    "B13F4485-94E0-41CD-AF71-913095D62E31",  # Frítest.jpg
     "1793FAAB-DE75-4E25-886C-2BD66C780D6A",  # Frítest.jpg
+    "35329C57-B963-48D6-BB75-6AFF9370CBBC",
+    "52083079-73D5-4921-AC1B-FE76F279133F",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
+    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "8846E3E6-8AC8-4857-8448-E3D025784410",
+    "A1DD1F98-2ECD-431F-9AC9-5AFEFE2D3A5C",
     "A8266C97-9BAF-4AF4-99F3-0013832869B8",  # Frítest.jpg
+    "B13F4485-94E0-41CD-AF71-913095D62E31",  # Frítest.jpg
+    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3",
     "D1D4040D-D141-44E8-93EA-E403D9F63E07",  # Frítest.jpg
+    "DC99FBDD-7A52-4100-A5BB-344131646C30",
+    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0",
 ]
 
 UUID_DUPLICATES = [
-    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
     "52083079-73D5-4921-AC1B-FE76F279133F",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48",
+    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0",
 ]
 
 UUID_LOCATION = "D79B8D77-BFFC-460B-9312-034F2877D35B"  # Pumkins2.jpg
@@ -2517,7 +2560,8 @@ def test_export_duplicate():
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
         result = runner.invoke(
-            export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--duplicate"]
+            export,
+            [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--duplicate", "--skip-raw"],
         )
         assert result.exit_code == 0
         files = glob.glob("*")
@@ -5084,7 +5128,7 @@ def test_export_dry_run():
             in result.output
         )
         for filepath in CLI_EXPORT_FILENAMES_DRY_RUN:
-            assert re.search(r"Exported.*" + f"{filepath}", result.output)
+            assert re.search(r"Exported.*" + f"{re.escape(filepath)}", result.output)
             assert not os.path.isfile(normalize_fs_path(filepath))
 
 
@@ -6070,6 +6114,69 @@ def test_export_cleanup_accented_album_name():
         assert "Deleted: 0 files, 0 directories" in result.output
 
 
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_cleanup_exiftool_accented_album_name_same_filenames():
+    """test export with --cleanup flag and photos in album with accented unicode characters (#561, #618)"""
+    import pathlib
+
+    from osxphotos.cli import export
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with tempfile.TemporaryDirectory() as tempdir:
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                tempdir,
+                "-V",
+                "--cleanup",
+                "--directory",
+                "{album[/,.|:,.]}",
+                "--exiftool",
+                "--exiftool-merge-keywords",
+                "--exiftool-merge-persons",
+                "--keyword-template",
+                "{keyword}",
+                "--report",
+                "test.csv",
+                "--skip-original-if-edited",
+                "--update",
+                "--touch-file",
+                "--not-hidden",
+            ],
+        )
+        assert "Deleted: 0 files, 0 directories" in result.output
+
+        # do it again
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                tempdir,
+                "-V",
+                "--cleanup",
+                "--directory",
+                "{album[/,.|:,.]}",
+                "--exiftool",
+                "--exiftool-merge-keywords",
+                "--exiftool-merge-persons",
+                "--keyword-template",
+                "{keyword}",
+                "--report",
+                "test.csv",
+                "--skip-original-if-edited",
+                "--update",
+                "--touch-file",
+                "--not-hidden",
+            ],
+        )
+        assert "exported: 0, updated: 0" in result.output
+        assert "updated EXIF data: 0" in result.output
+        assert "Deleted: 0 files, 0 directories" in result.output
+
+
 def test_save_load_config():
     """test --save-config, --load-config"""
     import glob