Fix for --name searching only original_filename on Photos 5+, #594

CHANGELOG.md

@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.45.5](https://github.com/RhetTbull/osxphotos/compare/v0.45.4...v0.45.5)
+
+> 5 February 2022
+
+- Fix for #561, no really, I mean it this time [`b3d3e14`](https://github.com/RhetTbull/osxphotos/commit/b3d3e14ffe41fbb22edb614b24f3985f379766a2)
+- Updated docs [skip ci] [`2b9ea11`](https://github.com/RhetTbull/osxphotos/commit/2b9ea11701799af9a661a8e2af70fca97235f487)
+- Updated tests for #561 [skip ci] [`77a49a0`](https://github.com/RhetTbull/osxphotos/commit/77a49a09a1bee74113a7114c543fbc25fa410ffc)
+
+#### [v0.45.4](https://github.com/RhetTbull/osxphotos/compare/v0.45.3...v0.45.4)
+
+> 3 February 2022
+
+- docs: add oPromessa as a contributor for ideas, test [`#611`](https://github.com/RhetTbull/osxphotos/pull/611)
+- Fix for filenames with special characters, #561, #618 [`f3063d3`](https://github.com/RhetTbull/osxphotos/commit/f3063d35be3c96342d83dbd87ddd614a2001bff4)
+- Updated docs [skip ci] [`06c5bbf`](https://github.com/RhetTbull/osxphotos/commit/06c5bbfcfdf591a4a5d43f1456adaa27385fe01a)
+- Added progress counter, #601 [`7ab5007`](https://github.com/RhetTbull/osxphotos/commit/7ab500740b28594dcd778140e10991f839220e9d)
+- Updated known issues [skip ci] [`e32090b`](https://github.com/RhetTbull/osxphotos/commit/e32090bf39cb786171b49443f878ffdbab774420)
+
 #### [v0.45.3](https://github.com/RhetTbull/osxphotos/compare/v0.45.2...v0.45.3)
 
 > 29 January 2022

README.md

@@ -1725,7 +1725,7 @@ Substitution                    Description
 {lf}                            A line feed: '\n', alias for {newline}
 {cr}                            A carriage return: '\r'
 {crlf}                          a carriage return + line feed: '\r\n'
-{osxphotos_version}             The osxphotos version, e.g. '0.45.4'
+{osxphotos_version}             The osxphotos version, e.g. '0.45.6'
 {osxphotos_cmd_line}            The full command line used to run osxphotos
 
 The following substitutions may result in multiple values. Thus if specified for
@@ -3629,7 +3629,7 @@ The following template field substitutions are availabe for use the templating s
 |{lf}|A line feed: '\n', alias for {newline}|
 |{cr}|A carriage return: '\r'|
 |{crlf}|a carriage return + line feed: '\r\n'|
-|{osxphotos_version}|The osxphotos version, e.g. '0.45.4'|
+|{osxphotos_version}|The osxphotos version, e.g. '0.45.6'|
 |{osxphotos_cmd_line}|The full command line used to run osxphotos|
 |{album}|Album(s) photo is contained in|
 |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 001a184f6f166bf8f64bf9bb56e7b73e
+config: 3c4bdd115410fda411407689f33d7c4c
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '0.45.4',
+    VERSION: '0.45.6',
     LANGUAGE: 'None',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

docs/cli.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/genindex.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Index &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Index &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/index.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/modules.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/reference.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos package &#8212; osxphotos 0.45.4 documentation</title>
+    <title>osxphotos package &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/search.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Search &#8212; osxphotos 0.45.4 documentation</title>
+    <title>Search &#8212; osxphotos 0.45.6 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.45.4"
+__version__ = "0.45.6"

osxphotos/export_db.py

@@ -10,13 +10,17 @@ import sys
 from abc import ABC, abstractmethod
 from io import StringIO
 from sqlite3 import Error
+from typing import Union
 
 from ._constants import OSXPHOTOS_EXPORT_DB
 from ._version import __version__
+from .utils import normalize_fs_path
 
 __all__ = ["ExportDB_ABC", "ExportDBNoOp", "ExportDB", "ExportDBInMemory"]
 
-OSXPHOTOS_EXPORTDB_VERSION = "4.2"
+OSXPHOTOS_EXPORTDB_VERSION = "4.3"
+OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH = "4.3"
+
 OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
 
 
@@ -211,12 +215,13 @@ class ExportDB(ExportDB_ABC):
         """query database for filename and return UUID
         returns None if filename not found in database
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filepath_normalized = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
             c.execute(
-                "SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,)
+                "SELECT uuid FROM files WHERE filepath_normalized = ?",
+                (filepath_normalized,),
             )
             results = c.fetchone()
             uuid = results[0] if results else None
@@ -228,7 +233,7 @@ class ExportDB(ExportDB_ABC):
     def set_uuid_for_file(self, filename, uuid):
         """set UUID of filename to uuid in the database"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -245,7 +250,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -266,7 +271,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -302,7 +307,7 @@ class ExportDB(ExportDB_ABC):
         """set stat info for filename (after exiftool has updated it)
         filename: filename to set the stat info for
         stat: a tuple of length 3: mode, size, mtime"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -323,7 +328,7 @@ class ExportDB(ExportDB_ABC):
         """get stat info for filename (after exiftool has updated it)
         returns: tuple of (mode, size, mtime)
         """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -384,7 +389,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_exifdata_for_file(self, filename):
         """returns the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -402,7 +407,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_exifdata_for_file(self, filename, exifdata):
         """sets the exifdata JSON struct for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -416,7 +421,7 @@ class ExportDB(ExportDB_ABC):
 
     def get_sidecar_for_file(self, filename):
         """returns the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -444,7 +449,7 @@ class ExportDB(ExportDB_ABC):
 
     def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
         """sets the sidecar data and signature for a file"""
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -515,7 +520,7 @@ class ExportDB(ExportDB_ABC):
     ):
         """sets all the data for file and uuid at once; if any value is None, does not set it"""
         filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
+        filename_normalized = self._normalize_filepath(filename)
         conn = self._conn
         try:
             c = conn.cursor()
@@ -577,7 +582,7 @@ class ExportDB(ExportDB_ABC):
             logging.warning(e)
 
     def _set_stat_for_file(self, table, filename, stats):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         if len(stats) != 3:
             raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
 
@@ -590,7 +595,7 @@ class ExportDB(ExportDB_ABC):
         conn.commit()
 
     def _get_stat_for_file(self, table, filename):
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        filename = self._normalize_filepath_relative(filename)
         conn = self._conn
         c = conn.cursor()
         c.execute(
@@ -626,6 +631,8 @@ class ExportDB(ExportDB_ABC):
             version_info = self._get_database_version(conn)
             if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION:
                 self._create_db_tables(conn)
+                if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH:
+                    self._migrate_normalized_filepath(conn)
                 self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
             else:
                 self.was_upgraded = ()
@@ -782,6 +789,32 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+    def _normalize_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, lower case"""
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _normalize_filepath_relative(self, filepath: Union[str, pathlib.Path]) -> str:
+        """normalize filepath for unicode, relative path (to export dir), lower case"""
+        filepath = str(pathlib.Path(filepath).relative_to(self._path))
+        return normalize_fs_path(str(filepath)).lower()
+
+    def _migrate_normalized_filepath(self, conn):
+        """Fix all filepath_normalized columns for unicode normalization"""
+        # Prior to database version 4.3, filepath_normalized was not normalized for unicode
+        c = conn.cursor()
+        for table in ["converted", "edited", "exifdata", "files", "sidecar"]:
+            old_values = c.execute(
+                f"SELECT filepath_normalized, id FROM {table}"
+            ).fetchall()
+            new_values = [
+                (self._normalize_filepath(filepath_normalized), id_)
+                for filepath_normalized, id_ in old_values
+            ]
+            c.executemany(
+                f"UPDATE {table} SET filepath_normalized=? WHERE id=?", new_values
+            )
+        conn.commit()
+
 
 class ExportDBInMemory(ExportDB):
     """In memory version of ExportDB

osxphotos/photosdb/photosdb.py

@@ -39,6 +39,7 @@ from .._constants import (
     _PHOTOS_5_PROJECT_ALBUM_KIND,
     _PHOTOS_5_ROOT_FOLDER_KIND,
     _PHOTOS_5_SHARED_ALBUM_KIND,
+    _PHOTOS_5_VERSION,
     _TESTED_OS_VERSIONS,
     _UNKNOWN_PERSON,
     BURST_KEY,
@@ -659,14 +660,18 @@ class PhotosDB:
 
         for person in c:
             pk = person[0]
-            fullname = person[2] if person[2] is not None else _UNKNOWN_PERSON
+            fullname = (
+                normalize_unicode(person[2])
+                if person[2] is not None
+                else _UNKNOWN_PERSON
+            )
             self._dbpersons_pk[pk] = {
                 "pk": pk,
                 "uuid": person[1],
                 "fullname": fullname,
                 "facecount": person[3],
                 "keyface": person[5],
-                "displayname": person[4],
+                "displayname": normalize_unicode(person[4]),
                 "photo_uuid": None,
                 "keyface_uuid": None,
             }
@@ -733,13 +738,6 @@ class PhotosDB:
             except KeyError:
                 self._dbfaces_pk[pk] = [uuid]
 
-        if _debug():
-            logging.debug(f"Finished walking through persons")
-            logging.debug(pformat(self._dbpersons_pk))
-            logging.debug(pformat(self._dbpersons_fullname))
-            logging.debug(pformat(self._dbfaces_pk))
-            logging.debug(pformat(self._dbfaces_uuid))
-
         # Get info on albums
         verbose("Processing albums.")
         c.execute(
@@ -876,14 +874,6 @@ class PhotosDB:
             else:
                 self._dbalbum_folders[album] = {}
 
-        if _debug():
-            logging.debug(f"Finished walking through albums")
-            logging.debug(pformat(self._dbalbums_album))
-            logging.debug(pformat(self._dbalbums_uuid))
-            logging.debug(pformat(self._dbalbum_details))
-            logging.debug(pformat(self._dbalbum_folders))
-            logging.debug(pformat(self._dbfolder_details))
-
         # Get info on keywords
         verbose("Processing keywords.")
         c.execute(
@@ -899,13 +889,16 @@ class PhotosDB:
                 RKMaster.uuid = RKVersion.masterUuid
             """
         )
-        for keyword in c:
+        for keyword_title, keyword_uuid, _ in c:
-            if not keyword[1] in self._dbkeywords_uuid:
+            keyword_title = normalize_unicode(keyword_title)
-                self._dbkeywords_uuid[keyword[1]] = []
+            try:
-            if not keyword[0] in self._dbkeywords_keyword:
+                self._dbkeywords_uuid[keyword_uuid].append(keyword_title)
-                self._dbkeywords_keyword[keyword[0]] = []
+            except KeyError:
-            self._dbkeywords_uuid[keyword[1]].append(keyword[0])
+                self._dbkeywords_uuid[keyword_uuid] = [keyword_title]
-            self._dbkeywords_keyword[keyword[0]].append(keyword[1])
+            try:
+                self._dbkeywords_keyword[keyword_title].append(keyword_uuid)
+            except KeyError:
+                self._dbkeywords_keyword[keyword_title] = [keyword_uuid]
 
         # Get info on disk volumes
         c.execute("select RKVolume.modelId, RKVolume.name from RKVolume")
@@ -1027,13 +1020,11 @@ class PhotosDB:
 
         for row in c:
             uuid = row[0]
-            if _debug():
-                logging.debug(f"uuid = '{uuid}, master = '{row[2]}")
             self._dbphotos[uuid] = {}
             self._dbphotos[uuid]["_uuid"] = uuid  # stored here for easier debugging
             self._dbphotos[uuid]["modelID"] = row[1]
             self._dbphotos[uuid]["masterUuid"] = row[2]
-            self._dbphotos[uuid]["filename"] = row[3]
+            self._dbphotos[uuid]["filename"] = normalize_unicode(row[3])
 
             # There are sometimes negative values for lastmodifieddate in the database
             # I don't know what these mean but they will raise exception in datetime if
@@ -1272,13 +1263,13 @@ class PhotosDB:
             info["volumeId"] = row[1]
             info["imagePath"] = row[2]
             info["isMissing"] = row[3]
-            info["originalFilename"] = row[4]
+            info["originalFilename"] = normalize_unicode(row[4])
             info["UTI"] = row[5]
             info["modelID"] = row[6]
             info["fileSize"] = row[7]
             info["isTrulyRAW"] = row[8]
             info["alternateMasterUuid"] = row[9]
-            info["filename"] = row[10]
+            info["filename"] = normalize_unicode(row[10])
             self._dbphotos_master[uuid] = info
 
         # get details needed to find path of the edited photos
@@ -1550,39 +1541,6 @@ class PhotosDB:
 
         # done processing, dump debug data if requested
         verbose("Done processing details from Photos library.")
-        if _debug():
-            logging.debug("Faces (_dbfaces_uuid):")
-            logging.debug(pformat(self._dbfaces_uuid))
-
-            logging.debug("Persons (_dbpersons_pk):")
-            logging.debug(pformat(self._dbpersons_pk))
-
-            logging.debug("Keywords by uuid (_dbkeywords_uuid):")
-            logging.debug(pformat(self._dbkeywords_uuid))
-
-            logging.debug("Keywords by keyword (_dbkeywords_keywords):")
-            logging.debug(pformat(self._dbkeywords_keyword))
-
-            logging.debug("Albums by uuid (_dbalbums_uuid):")
-            logging.debug(pformat(self._dbalbums_uuid))
-
-            logging.debug("Albums by album (_dbalbums_albums):")
-            logging.debug(pformat(self._dbalbums_album))
-
-            logging.debug("Album details (_dbalbum_details):")
-            logging.debug(pformat(self._dbalbum_details))
-
-            logging.debug("Album titles (_dbalbum_titles):")
-            logging.debug(pformat(self._dbalbum_titles))
-
-            logging.debug("Volumes (_dbvolumes):")
-            logging.debug(pformat(self._dbvolumes))
-
-            logging.debug("Photos (_dbphotos):")
-            logging.debug(pformat(self._dbphotos))
-
-            logging.debug("Burst Photos (dbphotos_burst:")
-            logging.debug(pformat(self._dbphotos_burst))
 
     def _build_album_folder_hierarchy_4(self, uuid, folders=None):
         """recursively build folder/album hierarchy
@@ -1673,7 +1631,7 @@ class PhotosDB:
         for person in c:
             pk = person[0]
             fullname = (
-                person[2]
+                normalize_unicode(person[2])
                 if (person[2] != "" and person[2] is not None)
                 else _UNKNOWN_PERSON
             )
@@ -1683,7 +1641,7 @@ class PhotosDB:
                 "fullname": fullname,
                 "facecount": person[3],
                 "keyface": person[4],
-                "displayname": person[5],
+                "displayname": normalize_unicode(person[5]),
                 "photo_uuid": None,
                 "keyface_uuid": None,
             }
@@ -1747,13 +1705,6 @@ class PhotosDB:
             except KeyError:
                 self._dbfaces_pk[pk] = [uuid]
 
-        if _debug():
-            logging.debug(f"Finished walking through persons")
-            logging.debug(pformat(self._dbpersons_pk))
-            logging.debug(pformat(self._dbpersons_fullname))
-            logging.debug(pformat(self._dbfaces_pk))
-            logging.debug(pformat(self._dbfaces_uuid))
-
         # get details about albums
         verbose("Processing albums.")
         c.execute(
@@ -1870,13 +1821,6 @@ class PhotosDB:
                 # shared albums can't be in folders
                 self._dbalbum_folders[album] = []
 
-        if _debug():
-            logging.debug(f"Finished walking through albums")
-            logging.debug(pformat(self._dbalbums_album))
-            logging.debug(pformat(self._dbalbums_uuid))
-            logging.debug(pformat(self._dbalbum_details))
-            logging.debug(pformat(self._dbalbum_folders))
-
         # get details on keywords
         verbose("Processing keywords.")
         c.execute(
@@ -1886,29 +1830,22 @@ class PhotosDB:
                 JOIN Z_1KEYWORDS ON Z_1KEYWORDS.Z_1ASSETATTRIBUTES = ZADDITIONALASSETATTRIBUTES.Z_PK 
                 JOIN ZKEYWORD ON ZKEYWORD.Z_PK = {keyword_join} """
         )
-        for keyword in c:
+        for keyword_title, keyword_uuid in c:
-            keyword_title = normalize_unicode(keyword[0])
+            keyword_title = normalize_unicode(keyword_title)
-            if not keyword[1] in self._dbkeywords_uuid:
+            try:
-                self._dbkeywords_uuid[keyword[1]] = []
+                self._dbkeywords_uuid[keyword_uuid].append(keyword_title)
-            if not keyword_title in self._dbkeywords_keyword:
+            except KeyError:
-                self._dbkeywords_keyword[keyword_title] = []
+                self._dbkeywords_uuid[keyword_uuid] = [keyword_title]
-            self._dbkeywords_uuid[keyword[1]].append(keyword[0])
+            try:
-            self._dbkeywords_keyword[keyword_title].append(keyword[1])
+                self._dbkeywords_keyword[keyword_title].append(keyword_uuid)
-
+            except KeyError:
-        if _debug():
+                self._dbkeywords_keyword[keyword_title] = [keyword_uuid]
-            logging.debug(f"Finished walking through keywords")
-            logging.debug(pformat(self._dbkeywords_keyword))
-            logging.debug(pformat(self._dbkeywords_uuid))
 
         # get details on disk volumes
         c.execute("SELECT ZUUID, ZNAME from ZFILESYSTEMVOLUME")
         for vol in c:
             self._dbvolumes[vol[0]] = vol[1]
 
-        if _debug():
-            logging.debug(f"Finished walking through volumes")
-            logging.debug(self._dbvolumes)
-
         # get details about photos
         verbose("Processing photo details.")
         c.execute(
@@ -2042,8 +1979,8 @@ class PhotosDB:
 
             info["hidden"] = row[9]
             info["favorite"] = row[10]
-            info["originalFilename"] = row[3]
+            info["originalFilename"] = normalize_unicode(row[3])
-            info["filename"] = row[12]
+            info["filename"] = normalize_unicode(row[12])
             info["directory"] = row[11]
 
             # set latitude and longitude
@@ -2521,48 +2458,6 @@ class PhotosDB:
 
         # done processing, dump debug data if requested
         verbose("Done processing details from Photos library.")
-        if _debug():
-            logging.debug("Faces (_dbfaces_uuid):")
-            logging.debug(pformat(self._dbfaces_uuid))
-
-            logging.debug("Persons (_dbpersons_pk):")
-            logging.debug(pformat(self._dbpersons_pk))
-
-            logging.debug("Keywords by uuid (_dbkeywords_uuid):")
-            logging.debug(pformat(self._dbkeywords_uuid))
-
-            logging.debug("Keywords by keyword (_dbkeywords_keywords):")
-            logging.debug(pformat(self._dbkeywords_keyword))
-
-            logging.debug("Albums by uuid (_dbalbums_uuid):")
-            logging.debug(pformat(self._dbalbums_uuid))
-
-            logging.debug("Albums by album (_dbalbums_albums):")
-            logging.debug(pformat(self._dbalbums_album))
-
-            logging.debug("Album details (_dbalbum_details):")
-            logging.debug(pformat(self._dbalbum_details))
-
-            logging.debug("Album titles (_dbalbum_titles):")
-            logging.debug(pformat(self._dbalbum_titles))
-
-            logging.debug("Album folders (_dbalbum_folders):")
-            logging.debug(pformat(self._dbalbum_folders))
-
-            logging.debug("Album parent folders (_dbalbum_parent_folders):")
-            logging.debug(pformat(self._dbalbum_parent_folders))
-
-            logging.debug("Albums pk (_dbalbums_pk):")
-            logging.debug(pformat(self._dbalbums_pk))
-
-            logging.debug("Volumes (_dbvolumes):")
-            logging.debug(pformat(self._dbvolumes))
-
-            logging.debug("Photos (_dbphotos):")
-            logging.debug(pformat(self._dbphotos))
-
-            logging.debug("Burst Photos (dbphotos_burst:")
-            logging.debug(pformat(self._dbphotos_burst))
 
     def _process_moments(self):
         """Process data from ZMOMENT table"""
@@ -2623,8 +2518,8 @@ class PhotosDB:
             moment_info["modificationDate"] = row[6]
             moment_info["representativeDate"] = row[7]
             moment_info["startDate"] = row[8]
-            moment_info["subtitle"] = row[9]
+            moment_info["subtitle"] = normalize_unicode(row[9])
-            moment_info["title"] = row[10]
+            moment_info["title"] = normalize_unicode(row[10])
             moment_info["uuid"] = row[11]
 
             # if both lat/lon == -180, then it means location undefined
@@ -3027,6 +2922,7 @@ class PhotosDB:
             if keywords:
                 keyword_set = set()
                 for keyword in keywords:
+                    keyword = normalize_unicode(keyword)
                     if keyword in self._dbkeywords_keyword:
                         keyword_set.update(self._dbkeywords_keyword[keyword])
                 photos_sets.append(keyword_set)
@@ -3034,6 +2930,7 @@ class PhotosDB:
             if persons:
                 person_set = set()
                 for person in persons:
+                    person = normalize_unicode(person)
                     if person in self._dbpersons_fullname:
                         for pk in self._dbpersons_fullname[person]:
                             try:
@@ -3076,8 +2973,6 @@ class PhotosDB:
                 ):
                     info = PhotoInfo(db=self, uuid=p, info=self._dbphotos[p])
                     photoinfo.append(info)
-        if _debug:
-            logging.debug(f"photoinfo: {pformat(photoinfo)}")
 
         return photoinfo
 
@@ -3414,23 +3309,35 @@ class PhotosDB:
                 # case-insensitive
                 for n in name:
                     n = n.lower()
-                    photo_list.extend(
+                    if self._db_version >= _PHOTOS_5_VERSION:
-                        [
+                        # search only original_filename (#594)
-                            p
+                        photo_list.extend(
-                            for p in photos
+                            [p for p in photos if n in p.original_filename.lower()]
-                            if n in p.filename.lower()
+                        )
-                            or n in p.original_filename.lower()
+                    else:
-                        ]
+                        photo_list.extend(
-                    )
+                            [
+                                p
+                                for p in photos
+                                if n in p.filename.lower()
+                                or n in p.original_filename.lower()
+                            ]
+                        )
             else:
                 for n in name:
-                    photo_list.extend(
+                    if self._db_version >= _PHOTOS_5_VERSION:
-                        [
+                        # search only original_filename (#594)
-                            p
+                        photo_list.extend(
-                            for p in photos
+                            [p for p in photos if n in p.original_filename]
-                            if n in p.filename or n in p.original_filename
+                        )
-                        ]
+                    else:
-                    )
+                        photo_list.extend(
+                            [
+                                p
+                                for p in photos
+                                if n in p.filename or n in p.original_filename
+                            ]
+                        )
             photos = photo_list
 
         if options.min_size:

tests/Test-10.15.7.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>585B80BF-8D1F-55EF-A9E8-6CF4E5523959</string>
 	<key>pid</key>
-	<integer>1961</integer>
+	<integer>14817</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.7.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2021-09-14T04:40:41Z</date>
+	<date>2022-02-04T13:51:39Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2021-07-20T05:48:08Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2021-07-20T05:47:59Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2021-09-14T04:40:43Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2021-09-14T04:40:42Z</date>
+	<date>2022-02-04T13:51:40Z</date>
 </dict>
 </plist>

tests/test_albums_folders_catalina_10_15_7.py

@@ -21,6 +21,7 @@ FOLDER_ALBUM_DICT = {
 ALBUM_NAMES = [
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum",
     "2019-10/11 Paris Clermont",
+    "Água",
     "AlbumInFolder",
     "EmptyAlbum",
     "I have a deleted twin",
@@ -38,6 +39,7 @@ ALBUM_NAMES = [
 ALBUM_PARENT_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": None,
     "2019-10/11 Paris Clermont": None,
+    "Água": None,
     "AlbumInFolder": "SubFolder2",
     "EmptyAlbum": None,
     "I have a deleted twin": None,
@@ -54,6 +56,7 @@ ALBUM_PARENT_DICT = {
 ALBUM_FOLDER_NAMES_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": [],
     "2019-10/11 Paris Clermont": [],
+    "Água": [],
     "AlbumInFolder": ["Folder1", "SubFolder2"],
     "EmptyAlbum": [],
     "I have a deleted twin": [],
@@ -70,6 +73,7 @@ ALBUM_FOLDER_NAMES_DICT = {
 ALBUM_LEN_DICT = {
     "2018-10 - Sponsion, Museum, Frühstück, Römermuseum": 1,
     "2019-10/11 Paris Clermont": 1,
+    "Água": 3,
     "AlbumInFolder": 2,
     "EmptyAlbum": 0,
     "I have a deleted twin": 1,
@@ -103,6 +107,11 @@ ALBUM_PHOTO_UUID_DICT = {
         "4D521201-92AC-43E5-8F7C-59BC41C37A96",
         "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
     ],
+    "Água": [
+        "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091",
+        "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
+        "54E76FCB-D353-4557-9997-0A457BCB4D48",
+    ],
 }
 
 UUID_DICT = {

tests/test_catalina_10_15_7.py

@@ -24,10 +24,10 @@ PHOTOS_DB = "tests/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_DB_PATH = "/Test-10.15.7.photoslibrary/database/photos.db"
 PHOTOS_LIBRARY_PATH = "/Test-10.15.7.photoslibrary"
 
-PHOTOS_DB_LEN = 25
+PHOTOS_DB_LEN = 29
-PHOTOS_NOT_IN_TRASH_LEN = 23
+PHOTOS_NOT_IN_TRASH_LEN = 27
 PHOTOS_IN_TRASH_LEN = 2
-PHOTOS_DB_IMPORT_SESSIONS = 17
+PHOTOS_DB_IMPORT_SESSIONS = 21
 
 KEYWORDS = [
     "Kids",
@@ -72,6 +72,7 @@ ALBUMS = [
     "Sorted Oldest First",
     "Sorted Title",
     "Test Album",  # there are 2 albums named "Test Album" for testing duplicate album names
+    "Água",
 ]
 KEYWORDS_DICT = {
     "Drink": 2,
@@ -115,6 +116,7 @@ ALBUM_DICT = {
     "Sorted Oldest First": 3,
     "Sorted Title": 3,
     "Test Album": 2,
+    "Água": 3,
 }  # Note: there are 2 albums named "Test Album" for testing duplicate album names
 
 UUID_DICT = {
@@ -1091,7 +1093,7 @@ def test_from_to_date(photosdb):
     time.tzset()
 
     photos = photosdb.photos(from_date=datetime.datetime(2018, 10, 28))
-    assert len(photos) == 16
+    assert len(photos) == 20
 
     photos = photosdb.photos(to_date=datetime.datetime(2018, 10, 28))
     assert len(photos) == 7

tests/test_cli.py

@@ -8,6 +8,7 @@ from click.testing import CliRunner
 
 import osxphotos
 from osxphotos.exiftool import get_exiftool_path
+from osxphotos.utils import normalize_unicode
 
 CLI_PHOTOS_DB = "tests/Test-10.15.7.photoslibrary"
 LIVE_PHOTOS_DB = "tests/Test-Cloud-10.15.1.photoslibrary"
@@ -79,64 +80,69 @@ CLI_OUTPUT_NO_SUBCOMMAND = [
 CLI_OUTPUT_QUERY_UUID = '[{"uuid": "D79B8D77-BFFC-460B-9312-034F2877D35B", "filename": "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg", "original_filename": "Pumkins2.jpg", "date": "2018-09-28T16:07:07-04:00", "description": "Girl holding pumpkin", "title": "I found one!", "keywords": ["Kids"], "albums": ["Pumpkin Farm", "Test Album", "Multi Keyword"], "persons": ["Katie"], "path": "/tests/Test-10.15.7.photoslibrary/originals/D/D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 41.256566, "longitude": -95.940257, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null}]'
 
 CLI_EXPORT_FILENAMES = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 
 CLI_EXPORT_FILENAMES_DRY_RUN = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
     "IMG_1693.tif",
-    "IMG_1994.JPG",
     "IMG_1994.cr2",
-    "IMG_1997.JPG",
+    "IMG_1994.JPG",
     "IMG_1997.cr2",
-    "IMG_3092.heic",
+    "IMG_1997.JPG",
     "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
     "IMG_4547.jpg",
     "Jellyfish.MOV",
     "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
     "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
     "winebottle.jpeg",
     "winebottle.jpeg",
-    "Frítest.jpg",
-    "Frítest_edited.jpeg",
 ]
 
 CLI_EXPORT_IGNORE_SIGNATURE_FILENAMES = ["Tulips.jpg", "wedding.jpg"]
@@ -154,225 +160,253 @@ CLI_EXPORT_ORIGINAL_SUFFIX_TEMPLATE = "{edited?_original,}"
 CLI_EXPORT_PREVIEW_SUFFIX = "_lowres"
 
 CLI_EXPORT_FILENAMES_EDITED_SUFFIX = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_bearbeiten.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_bearbeiten.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_bearbeiten.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_bearbeiten.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_bearbeiten.jpeg",
     "Frítest_bearbeiten (1).jpeg",
+    "Frítest_bearbeiten.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_bearbeiten.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_bearbeiten.jpeg",
+    "St James Park.jpg",
+    "Tulips_bearbeiten.jpeg",
+    "Tulips.jpg",
+    "wedding_bearbeiten.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_EDITED_SUFFIX_TEMPLATE = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
     "DSC03584.dng",
-    "IMG_1693.tif",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.tif",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.heic",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_ORIGINAL_SUFFIX = [
-    "Pumkins1_original.jpg",
+    "[2020-08-29] AAF035_original (1).jpg",
-    "Pumkins2_original.jpg",
+    "[2020-08-29] AAF035_original (2).jpg",
-    "Pumpkins3_original.jpg",
+    "[2020-08-29] AAF035_original (3).jpg",
-    "St James Park_original.jpg",
+    "[2020-08-29] AAF035_original.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips_original.jpg",
-    "wedding_original.jpg",
-    "wedding_edited.jpeg",
     "DSC03584_original.dng",
-    "IMG_1693_original.tif",
+    "Frítest_edited (1).jpeg",
-    "IMG_1994_original.JPG",
+    "Frítest_edited.jpeg",
-    "IMG_1994_original.cr2",
-    "IMG_1997_original.JPG",
-    "IMG_1997_original.cr2",
-    "IMG_3092_original.heic",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547_original.jpg",
-    "Jellyfish_original.MOV",
-    "Jellyfish1_original.mp4",
-    "Tulips_edited.jpeg",
-    "screenshot-really-a-png_original.jpeg",
-    "winebottle_original.jpeg",
-    "winebottle_original (1).jpeg",
-    "Frítest_original.jpg",
     "Frítest_original (1).jpg",
     "Frítest_original (2).jpg",
     "Frítest_original (3).jpg",
-    "Frítest_edited.jpeg",
+    "Frítest_original.jpg",
-    "Frítest_edited (1).jpeg",
+    "IMG_1693_original.tif",
+    "IMG_1994_original.cr2",
+    "IMG_1994_original.JPG",
+    "IMG_1997_original.cr2",
+    "IMG_1997_original.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092_original.heic",
+    "IMG_4547_original.jpg",
+    "Jellyfish_original.MOV",
+    "Jellyfish1_original.mp4",
+    "Pumkins1_original.jpg",
+    "Pumkins2_original.jpg",
+    "Pumpkins3_original.jpg",
+    "screenshot-really-a-png_original.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park_original.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips_original.jpg",
+    "wedding_edited.jpeg",
+    "wedding_original.jpg",
+    "winebottle_original (1).jpeg",
+    "winebottle_original.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_ORIGINAL_SUFFIX_TEMPLATE = [
-    "Pumkins1.jpg",
+    "[2020-08-29] AAF035 (1).jpg",
-    "Pumkins2.jpg",
+    "[2020-08-29] AAF035 (2).jpg",
-    "Pumpkins3.jpg",
+    "[2020-08-29] AAF035 (3).jpg",
-    "St James Park_original.jpg",
+    "[2020-08-29] AAF035.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips_original.jpg",
-    "wedding_original.jpg",
-    "wedding_edited.jpeg",
-    "Tulips_edited.jpeg",
     "DSC03584.dng",
+    "Frítest (1).jpg",
+    "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest_original (1).jpg",
+    "Frítest_original.jpg",
+    "Frítest.jpg",
     "IMG_1693.tif",
-    "IMG_1994.JPG",
     "IMG_1994.cr2",
-    "IMG_1997.JPG",
+    "IMG_1994.JPG",
     "IMG_1997.cr2",
-    "IMG_3092_original.heic",
+    "IMG_1997.JPG",
     "IMG_3092_edited.jpeg",
+    "IMG_3092_original.heic",
     "IMG_4547.jpg",
     "Jellyfish.MOV",
     "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
     "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park_original.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips_original.jpg",
+    "wedding_edited.jpeg",
+    "wedding_original.jpg",
     "winebottle (1).jpeg",
-    "Frítest.jpg",
+    "winebottle.jpeg",
-    "Frítest (1).jpg",
-    "Frítest_original.jpg",
-    "Frítest_edited.jpeg",
-    "Frítest_original (1).jpg",
-    "Frítest_edited (1).jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CURRENT = [
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A_edited.jpeg",  # Frítest.jpg
+    "1793FAAB-DE75-4E25-886C-2BD66C780D6A.jpeg",  # Frítest.jpg
     "1EB2B765-0765-43BA-A90C-0D0580E6172C.jpeg",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A.jpeg",
+    "35329C57-B963-48D6-BB75-6AFF9370CBBC.mov",
     "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg",
     "4D521201-92AC-43E5-8F7C-59BC41C37A96.cr2",
     "4D521201-92AC-43E5-8F7C-59BC41C37A96.jpeg",
+    "52083079-73D5-4921-AC1B-FE76F279133F.jpeg",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48.jpeg",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4_edited.jpeg",
     "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4.jpeg",
+    "7783E8E6-9CAC-40F3-BE22-81FB7051C266_edited.jpeg",
+    "7783E8E6-9CAC-40F3-BE22-81FB7051C266.heic",
+    "7F74DD34-5920-4DA3-B284-479887A34F66.jpeg",
+    "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091.jpeg",
+    "8846E3E6-8AC8-4857-8448-E3D025784410.tiff",
+    "A8266C97-9BAF-4AF4-99F3-0013832869B8.jpeg",  # Frítest.jpg
     "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.cr2",
     "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.jpeg",
-    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068.dng",
-    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30.jpeg",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30_edited.jpeg",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51_edited.jpeg",
-    "F12384F6-CD17-4151-ACBA-AE0E3688539E.jpeg",
-    "35329C57-B963-48D6-BB75-6AFF9370CBBC.mov",
-    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4_edited.jpeg",
-    "7783E8E6-9CAC-40F3-BE22-81FB7051C266.heic",
-    "7783E8E6-9CAC-40F3-BE22-81FB7051C266_edited.jpeg",
-    "7F74DD34-5920-4DA3-B284-479887A34F66.jpeg",
-    "8846E3E6-8AC8-4857-8448-E3D025784410.tiff",
-    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3.mp4",
-    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91.jpeg",
-    "52083079-73D5-4921-AC1B-FE76F279133F.jpeg",
     "B13F4485-94E0-41CD-AF71-913095D62E31.jpeg",  # Frítest.jpg
-    "1793FAAB-DE75-4E25-886C-2BD66C780D6A.jpeg",  # Frítest.jpg
+    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068.dng",
-    "1793FAAB-DE75-4E25-886C-2BD66C780D6A_edited.jpeg",  # Frítest.jpg
+    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3.mp4",
-    "A8266C97-9BAF-4AF4-99F3-0013832869B8.jpeg",  # Frítest.jpg
-    "D1D4040D-D141-44E8-93EA-E403D9F63E07.jpeg",  # Frítest.jpg
     "D1D4040D-D141-44E8-93EA-E403D9F63E07_edited.jpeg",  # Frítest.jpg
+    "D1D4040D-D141-44E8-93EA-E403D9F63E07.jpeg",  # Frítest.jpg
+    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
+    "DC99FBDD-7A52-4100-A5BB-344131646C30_edited.jpeg",
+    "DC99FBDD-7A52-4100-A5BB-344131646C30.jpeg",
+    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91.jpeg",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51_edited.jpeg",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg",
+    "F12384F6-CD17-4151-ACBA-AE0E3688539E.jpeg",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CONVERT_TO_JPEG = [
+    "[2020-08-29] AAF035 (1).jpg",
+    "[2020-08-29] AAF035 (2).jpg",
+    "[2020-08-29] AAF035 (3).jpg",
+    "[2020-08-29] AAF035.jpg",
     "DSC03584.jpeg",
-    "IMG_1693.jpeg",
-    "IMG_1994.JPG",
-    "IMG_1994.cr2",
-    "IMG_1997.JPG",
-    "IMG_1997.cr2",
-    "IMG_3092.jpeg",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Pumkins1.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "Tulips_edited.jpeg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
     "Frítest_edited (1).jpeg",
     "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.jpeg",
+    "IMG_1994.cr2",
+    "IMG_1994.JPG",
+    "IMG_1997.cr2",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.jpeg",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_FILENAMES_CONVERT_TO_JPEG_SKIP_RAW = [
+    "[2020-08-29] AAF035 (1).jpg",
+    "[2020-08-29] AAF035 (2).jpg",
+    "[2020-08-29] AAF035 (3).jpg",
+    "[2020-08-29] AAF035.jpg",
     "DSC03584.jpeg",
-    "IMG_1693.jpeg",
-    "IMG_1994.JPG",
-    "IMG_1997.JPG",
-    "IMG_3092.jpeg",
-    "IMG_3092_edited.jpeg",
-    "IMG_4547.jpg",
-    "Pumkins1.jpg",
-    "Pumkins2.jpg",
-    "Pumpkins3.jpg",
-    "St James Park.jpg",
-    "St James Park_edited.jpeg",
-    "Tulips.jpg",
-    "Tulips_edited.jpeg",
-    "wedding.jpg",
-    "wedding_edited.jpeg",
-    "Jellyfish.MOV",
-    "Jellyfish1.mp4",
-    "screenshot-really-a-png.jpeg",
-    "winebottle.jpeg",
-    "winebottle (1).jpeg",
-    "Frítest.jpg",
     "Frítest (1).jpg",
     "Frítest (2).jpg",
     "Frítest (3).jpg",
-    "Frítest_edited.jpeg",
     "Frítest_edited (1).jpeg",
+    "Frítest_edited.jpeg",
+    "Frítest.jpg",
+    "IMG_1693.jpeg",
+    "IMG_1994.JPG",
+    "IMG_1997.JPG",
+    "IMG_3092_edited.jpeg",
+    "IMG_3092.jpeg",
+    "IMG_4547.jpg",
+    "Jellyfish.MOV",
+    "Jellyfish1.mp4",
+    "Pumkins1.jpg",
+    "Pumkins2.jpg",
+    "Pumpkins3.jpg",
+    "screenshot-really-a-png.jpeg",
+    "St James Park_edited.jpeg",
+    "St James Park.jpg",
+    "Tulips_edited.jpeg",
+    "Tulips.jpg",
+    "wedding_edited.jpeg",
+    "wedding.jpg",
+    "winebottle (1).jpeg",
+    "winebottle.jpeg",
 ]
 
 CLI_EXPORT_CONVERT_TO_JPEG_LARGE_FILE = "DSC03584.jpeg"
@@ -546,7 +580,7 @@ PHOTOS_NOT_IN_TRASH_LEN_14_6 = 12
 PHOTOS_IN_TRASH_LEN_14_6 = 1
 PHOTOS_MISSING_14_6 = 1
 
-PHOTOS_NOT_IN_TRASH_LEN_15_7 = 23
+PHOTOS_NOT_IN_TRASH_LEN_15_7 = 27
 PHOTOS_IN_TRASH_LEN_15_7 = 2
 PHOTOS_MISSING_15_7 = 2
 PHOTOS_EDITED_15_7 = 6
@@ -732,6 +766,7 @@ ALBUMS_JSON = {
         "Sorted Newest First": 3,
         "Sorted Oldest First": 3,
         "Sorted Title": 3,
+        "Água": 3,
     },
     "shared albums": {},
 }
@@ -746,6 +781,7 @@ ALBUMS_STR = """albums:
   2018-10 - Sponsion, Museum, Frühstück, Römermuseum: 1
   2019-10/11 Paris Clermont: 1
   EmptyAlbum: 0
+  Água: 3
 shared albums: {}
 """
 
@@ -820,37 +856,45 @@ UUID_IS_REFERENCE = [
 ]
 
 UUID_IN_ALBUM = [
-    "F12384F6-CD17-4151-ACBA-AE0E3688539E",
-    "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
     "1EB2B765-0765-43BA-A90C-0D0580E6172C",
-    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
-    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
-    "D79B8D77-BFFC-460B-9312-034F2877D35B",
-    "4D521201-92AC-43E5-8F7C-59BC41C37A96",
-    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068",
     "3DD2C897-F19E-4CA6-8C22-B027D5A71907",
+    "4D521201-92AC-43E5-8F7C-59BC41C37A96",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48",
     "7783E8E6-9CAC-40F3-BE22-81FB7051C266",
+    "7FD37B5F-6FAA-4DB1-8A29-BF9C37E38091",
+    "8E1D7BC9-9321-44F9-8CFB-4083F6B9232A",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
+    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068",
+    "D79B8D77-BFFC-460B-9312-034F2877D35B",
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51",
+    "F12384F6-CD17-4151-ACBA-AE0E3688539E",
 ]
 
 UUID_NOT_IN_ALBUM = [
-    "A1DD1F98-2ECD-431F-9AC9-5AFEFE2D3A5C",
-    "DC99FBDD-7A52-4100-A5BB-344131646C30",
-    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3",
-    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91",
-    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
-    "35329C57-B963-48D6-BB75-6AFF9370CBBC",
-    "8846E3E6-8AC8-4857-8448-E3D025784410",
-    "7F74DD34-5920-4DA3-B284-479887A34F66",
-    "52083079-73D5-4921-AC1B-FE76F279133F",
-    "B13F4485-94E0-41CD-AF71-913095D62E31",  # Frítest.jpg
     "1793FAAB-DE75-4E25-886C-2BD66C780D6A",  # Frítest.jpg
+    "35329C57-B963-48D6-BB75-6AFF9370CBBC",
+    "52083079-73D5-4921-AC1B-FE76F279133F",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
+    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "8846E3E6-8AC8-4857-8448-E3D025784410",
+    "A1DD1F98-2ECD-431F-9AC9-5AFEFE2D3A5C",
     "A8266C97-9BAF-4AF4-99F3-0013832869B8",  # Frítest.jpg
+    "B13F4485-94E0-41CD-AF71-913095D62E31",  # Frítest.jpg
+    "D1359D09-1373-4F3B-B0E3-1A4DE573E4A3",
     "D1D4040D-D141-44E8-93EA-E403D9F63E07",  # Frítest.jpg
+    "DC99FBDD-7A52-4100-A5BB-344131646C30",
+    "E2078879-A29C-4D6F-BACB-E3BBE6C3EB91",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0",
 ]
 
 UUID_DUPLICATES = [
-    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "2DFD33F1-A5D8-486F-A3A9-98C07995535A",
     "52083079-73D5-4921-AC1B-FE76F279133F",
+    "54E76FCB-D353-4557-9997-0A457BCB4D48",
+    "7F74DD34-5920-4DA3-B284-479887A34F66",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91",
+    "F207D5DE-EFAD-4217-8424-0764AAC971D0",
 ]
 
 UUID_LOCATION = "D79B8D77-BFFC-460B-9312-034F2877D35B"  # Pumkins2.jpg
@@ -2517,7 +2561,8 @@ def test_export_duplicate():
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
         result = runner.invoke(
-            export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--duplicate"]
+            export,
+            [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--duplicate", "--skip-raw"],
         )
         assert result.exit_code == 0
         files = glob.glob("*")
@@ -5084,7 +5129,7 @@ def test_export_dry_run():
             in result.output
         )
         for filepath in CLI_EXPORT_FILENAMES_DRY_RUN:
-            assert re.search(r"Exported.*" + f"{filepath}", result.output)
+            assert re.search(r"Exported.*" + f"{re.escape(filepath)}", result.output)
             assert not os.path.isfile(normalize_fs_path(filepath))
 
 
@@ -6029,7 +6074,7 @@ def test_export_cleanup_empty_album():
 
 
 def test_export_cleanup_accented_album_name():
-    """test export with --cleanup flag and photos in album with accented unicode characters (#561)"""
+    """test export with --cleanup flag and photos in album with accented unicode characters (#561, #618)"""
     import pathlib
 
     from osxphotos.cli import export
@@ -6052,6 +6097,86 @@ def test_export_cleanup_accented_album_name():
         )
         assert "Deleted: 0 files, 0 directories" in result.output
 
+        # do it again
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                tempdir,
+                "-V",
+                "--update",
+                "--cleanup",
+                "--directory",
+                "{folder_album}",
+                "--update",
+            ],
+        )
+        assert "exported: 0, updated: 0" in result.output
+        assert "Deleted: 0 files, 0 directories" in result.output
+
+
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_cleanup_exiftool_accented_album_name_same_filenames():
+    """test export with --cleanup flag and photos in album with accented unicode characters (#561, #618)"""
+    import pathlib
+
+    from osxphotos.cli import export
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with tempfile.TemporaryDirectory() as tempdir:
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                tempdir,
+                "-V",
+                "--cleanup",
+                "--directory",
+                "{album[/,.|:,.]}",
+                "--exiftool",
+                "--exiftool-merge-keywords",
+                "--exiftool-merge-persons",
+                "--keyword-template",
+                "{keyword}",
+                "--report",
+                "test.csv",
+                "--skip-original-if-edited",
+                "--update",
+                "--touch-file",
+                "--not-hidden",
+            ],
+        )
+        assert "Deleted: 0 files, 0 directories" in result.output
+
+        # do it again
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                tempdir,
+                "-V",
+                "--cleanup",
+                "--directory",
+                "{album[/,.|:,.]}",
+                "--exiftool",
+                "--exiftool-merge-keywords",
+                "--exiftool-merge-persons",
+                "--keyword-template",
+                "{keyword}",
+                "--report",
+                "test.csv",
+                "--skip-original-if-edited",
+                "--update",
+                "--touch-file",
+                "--not-hidden",
+            ],
+        )
+        assert "exported: 0, updated: 0" in result.output
+        assert "updated EXIF data: 0" in result.output
+        assert "Deleted: 0 files, 0 directories" in result.output
+
 
 def test_save_load_config():
     """test --save-config, --load-config"""
@@ -7008,6 +7133,30 @@ def test_query_name():
     assert json_got[0]["original_filename"] == "DSC03584.dng"
 
 
+def test_query_name_unicode():
+    """test query --name with a unicode name"""
+    import json
+    import os
+    import os.path
+
+    import osxphotos
+    from osxphotos.cli import query
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    result = runner.invoke(
+        query,
+        ["--json", "--db", os.path.join(cwd, PHOTOS_DB_15_7), "--name", "Frítest"],
+    )
+    assert result.exit_code == 0
+    json_got = json.loads(result.output)
+
+    assert len(json_got) == 4
+    assert normalize_unicode(json_got[0]["original_filename"]).startswith(
+        normalize_unicode("Frítest.jpg")
+    )
+
+
 def test_query_name_i():
     """test query --name -i"""
     import json
@@ -7037,6 +7186,46 @@ def test_query_name_i():
     assert json_got[0]["original_filename"] == "DSC03584.dng"
 
 
+def test_query_name_original_filename():
+    """test query --name only searches original filename on Photos 5+"""
+    import json
+    import os
+    import os.path
+
+    from osxphotos.cli import query
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    result = runner.invoke(
+        query,
+        ["--json", "--db", os.path.join(cwd, PHOTOS_DB_15_7), "--name", "AA"],
+    )
+    assert result.exit_code == 0
+    json_got = json.loads(result.output)
+
+    assert len(json_got) == 4
+
+
+def test_query_name_original_filename_i():
+    """test query --name only searches original filename on Photos 5+ with -i"""
+    import json
+    import os
+    import os.path
+
+    from osxphotos.cli import query
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    result = runner.invoke(
+        query,
+        ["--json", "--db", os.path.join(cwd, PHOTOS_DB_15_7), "--name", "aa", "-i"],
+    )
+    assert result.exit_code == 0
+    json_got = json.loads(result.output)
+
+    assert len(json_got) == 4
+
+
 def test_export_name():
     """test export --name"""
     import glob