Unicode refactor (#1101)

* Began refactoring for improving unicode handling

* Added platform and unicode modules

* Added tests for unicode utilities

* Added tests for unicode utilities

* Added tests for unicode utilities

* Added tests for unicode utilities

* Fixed unicode tests for linux

* Fixed unicode tests for linux

* Fixed duplicate alubm name with --add-to-album

* Fixed test for linux

* Fix for duplicate unicode kewyords, see #907, #1085

examples/xmp_rating.py

@@ -7,7 +7,7 @@ from typing import Callable
 
 from osxphotos import ExportResults, PhotoInfo
 from osxphotos.exiftool import ExifTool
-from osxphotos.utils import normalize_unicode
+from osxphotos.unicode import normalize_unicode
 
 # Update this for your custom keyword to rating mapping
 RATINGS = {

osxphotos/__init__.py

@@ -21,10 +21,10 @@ from .photosdb._photosdb_process_comments import CommentInfo, LikeInfo
 from .phototables import PhotoTables
 from .phototemplate import PhotoTemplate
 from .placeinfo import PlaceInfo
+from .platform import is_macos
 from .queryoptions import QueryOptions
 from .scoreinfo import ScoreInfo
 from .searchinfo import SearchInfo
-from .utils import is_macos
 
 if is_macos:
     from .photosalbum import PhotosAlbum, PhotosAlbumPhotoScript

osxphotos/_constants.py

@@ -18,9 +18,6 @@ OSXPHOTOS_URL = "https://github.com/RhetTbull/osxphotos"
 # Apple Epoch is Jan 1, 2001
 TIME_DELTA = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
 
-# Unicode format to use for comparing strings
-UNICODE_FORMAT = "NFC"
-
 # which Photos library database versions have been tested
 # Photos 2.0 (10.12.6) == 2622
 # Photos 3.0 (10.13.6) == 3301

osxphotos/cli/__init__.py

@@ -13,7 +13,7 @@ from osxphotos.debug import (
     set_debug,
     wrap_function,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 # apply any debug functions
 # need to do this before importing anything else so that the debug functions

osxphotos/cli/add_locations.py

@@ -7,8 +7,9 @@ import datetime
 import click
 
 import osxphotos
+from osxphotos.platform import assert_macos
 from osxphotos.queryoptions import IncompatibleQueryOptions, query_options_from_kwargs
-from osxphotos.utils import assert_macos, pluralize
+from osxphotos.utils import pluralize
 
 from .cli_params import QUERY_OPTIONS, THEME_OPTION, TIMESTAMP_OPTION, VERBOSE_OPTION
 from .click_rich_echo import rich_click_echo as echo

osxphotos/cli/batch_edit.py

@@ -12,8 +12,8 @@ import click
 
 import osxphotos
 from osxphotos.phototemplate import RenderOptions
+from osxphotos.platform import assert_macos
 from osxphotos.sqlitekvstore import SQLiteKVStore
-from osxphotos.utils import assert_macos
 
 assert_macos()
 

osxphotos/cli/cli.py

@@ -9,7 +9,7 @@ import click
 
 from osxphotos._constants import PROFILE_SORT_KEYS
 from osxphotos._version import __version__
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 from .about import about
 from .albums import albums

osxphotos/cli/cli_params.py

@@ -10,7 +10,7 @@ from typing import Any, Callable
 
 import click
 
-from ..utils import is_macos
+from ..platform import is_macos
 from .common import OSXPHOTOS_HIDDEN, print_version
 from .param_types import *
 

osxphotos/cli/common.py

@@ -15,7 +15,8 @@ from xdg import xdg_config_home, xdg_data_home
 import osxphotos
 from osxphotos._constants import APP_NAME
 from osxphotos._version import __version__
-from osxphotos.utils import get_latest_version, get_macos_version
+from osxphotos.platform import get_macos_version
+from osxphotos.utils import get_latest_version
 
 # used to show/hide hidden commands
 OSXPHOTOS_HIDDEN = not bool(os.getenv("OSXPHOTOS_SHOW_HIDDEN", default=False))

osxphotos/cli/darkmode.py

@@ -1,6 +1,6 @@
 """Detect dark mode on MacOS >= 10.14 or fake it elsewhere"""
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import Foundation

osxphotos/cli/export.py

@@ -45,16 +45,11 @@ from osxphotos.path_utils import is_valid_filepath, sanitize_filename, sanitize_
 from osxphotos.photoexporter import ExportOptions, ExportResults, PhotoExporter
 from osxphotos.photoinfo import PhotoInfoNone
 from osxphotos.phototemplate import PhotoTemplate, RenderOptions
+from osxphotos.platform import get_macos_version, is_macos
 from osxphotos.queryoptions import load_uuid_from_file, query_options_from_kwargs
+from osxphotos.unicode import normalize_fs_path
 from osxphotos.uti import get_preferred_uti_extension
-from osxphotos.utils import (
-    format_sec_to_hhmmss,
-    get_macos_version,
-    is_macos,
-    normalize_fs_path,
-    pluralize,
-    under_test,
-)
+from osxphotos.utils import format_sec_to_hhmmss, pluralize, under_test
 
 if is_macos:
     from osxmetadata import (

osxphotos/cli/help.py

@@ -20,7 +20,7 @@ from osxphotos.phototemplate import (
     TEMPLATE_SUBSTITUTIONS_PATHLIB,
     get_template_help,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     from osxmetadata import MDITEM_ATTRIBUTE_DATA, MDITEM_ATTRIBUTE_SHORT_NAMES

osxphotos/cli/import_cli.py

@@ -41,8 +41,10 @@ from osxphotos.exiftool import ExifToolCaching, get_exiftool_path
 from osxphotos.photoinfo import PhotoInfoNone
 from osxphotos.photosalbum import PhotosAlbumPhotoScript
 from osxphotos.phototemplate import PhotoTemplate, RenderOptions
+from osxphotos.platform import assert_macos
 from osxphotos.sqlitekvstore import SQLiteKVStore
-from osxphotos.utils import assert_macos, pluralize
+from osxphotos.unicode import normalize_unicode
+from osxphotos.utils import pluralize
 
 assert_macos()
 
@@ -357,11 +359,12 @@ def set_photo_metadata(
     merge_keywords: bool,
 ) -> MetaData:
     """Set metadata (title, description, keywords) for a Photo object"""
-    photo.title = metadata.title
-    photo.description = metadata.description
+    photo.title = normalize_unicode(metadata.title)
+    photo.description = normalize_unicode(metadata.description)
     keywords = metadata.keywords.copy()
+    keywords =normalize_unicode(keywords)
     if merge_keywords:
-        if old_keywords := photo.keywords:
+        if old_keywords := normalize_unicode(photo.keywords):
             keywords.extend(old_keywords)
             keywords = list(set(keywords))
     photo.keywords = keywords
@@ -420,7 +423,7 @@ def set_photo_title(
         verbose(
             f"Setting title of photo [filename]{filepath.name}[/] to '{title_text[0]}'"
         )
-        photo.title = title_text[0]
+        photo.title = normalize_unicode(title_text[0])
         return title_text[0]
     else:
         return ""
@@ -448,7 +451,7 @@ def set_photo_description(
         verbose(
             f"Setting description of photo [filename]{filepath.name}[/] to '{description_text[0]}'"
         )
-        photo.description = description_text[0]
+        photo.description = normalize_unicode(description_text[0])
         return description_text[0]
     else:
         return ""
@@ -469,8 +472,9 @@ def set_photo_keywords(
         kw = render_photo_template(filepath, relative_filepath, keyword, exiftool_path)
         keywords.extend(kw)
     if keywords:
+        keywords = normalize_unicode(keywords)
         if merge:
-            if old_keywords := photo.keywords:
+            if old_keywords := normalize_unicode(photo.keywords):
                 keywords.extend(old_keywords)
                 keywords = list(set(keywords))
         verbose(f"Setting keywords of photo [filename]{filepath.name}[/] to {keywords}")

osxphotos/cli/photo_inspect.py

@@ -20,8 +20,9 @@ from rich.panel import Panel
 
 from osxphotos import PhotoInfo, PhotosDB
 from osxphotos._constants import _UNKNOWN_PERSON, search_category_factory
+from osxphotos.platform import assert_macos
 from osxphotos.rich_utils import add_rich_markup_tag
-from osxphotos.utils import assert_macos, dd_to_dms_str
+from osxphotos.utils import dd_to_dms_str
 
 assert_macos()
 

osxphotos/cli/query.py

@@ -12,8 +12,8 @@ from osxphotos.cli.click_rich_echo import (
 )
 from osxphotos.debug import set_debug
 from osxphotos.phototemplate import RenderOptions
+from osxphotos.platform import assert_macos, is_macos
 from osxphotos.queryoptions import query_options_from_kwargs
-from osxphotos.utils import assert_macos, is_macos
 
 if is_macos:
     from osxphotos.photosalbum import PhotosAlbum

osxphotos/cli/repl.py

@@ -20,13 +20,13 @@ from osxphotos._constants import _PHOTOS_4_VERSION
 from osxphotos.cli.click_rich_echo import rich_echo_error as echo_error
 from osxphotos.photoinfo import PhotoInfo
 from osxphotos.photosdb import PhotosDB
+from osxphotos.platform import assert_macos, is_macos
 from osxphotos.pyrepl import embed_repl
 from osxphotos.queryoptions import (
     IncompatibleQueryOptions,
     QueryOptions,
     query_options_from_kwargs,
 )
-from osxphotos.utils import assert_macos, is_macos
 
 if is_macos:
     import photoscript

osxphotos/cli/show_command.py

@@ -8,7 +8,8 @@ import click
 from osxphotos._constants import UUID_PATTERN
 from osxphotos.export_db_utils import get_uuid_for_filepath
 from osxphotos.photosdb.photosdb_utils import get_photos_library_version
-from osxphotos.utils import assert_macos, get_last_library_path
+from osxphotos.platform import assert_macos
+from osxphotos.utils import get_last_library_path
 
 assert_macos()
 

osxphotos/cli/sync.py

@@ -15,13 +15,14 @@ from osxphotos.photoinfo import PhotoInfoNone
 from osxphotos.photosalbum import PhotosAlbum
 from osxphotos.photosdb.photosdb_utils import get_db_version
 from osxphotos.phototemplate import PhotoTemplate, RenderOptions
+from osxphotos.platform import assert_macos
 from osxphotos.queryoptions import (
     IncompatibleQueryOptions,
     QueryOptions,
     query_options_from_kwargs,
 )
 from osxphotos.sqlitekvstore import SQLiteKVStore
-from osxphotos.utils import assert_macos, pluralize
+from osxphotos.utils import pluralize
 
 assert_macos()
 

osxphotos/cli/timewarp.py

@@ -25,7 +25,8 @@ from osxphotos.photodates import (
     update_photo_time_for_new_timezone,
 )
 from osxphotos.phototz import PhotoTimeZone, PhotoTimeZoneUpdater
-from osxphotos.utils import assert_macos, noop, pluralize
+from osxphotos.platform import assert_macos
+from osxphotos.utils import noop, pluralize
 
 assert_macos()
 

osxphotos/cli/uuid.py

@@ -2,7 +2,7 @@
 
 import click
 
-from osxphotos.utils import assert_macos
+from osxphotos.platform import assert_macos
 
 assert_macos()
 

osxphotos/compare_exif.py

@@ -7,7 +7,8 @@ from osxphotos import PhotosDB
 from osxphotos.exiftool import ExifTool
 
 from .datetime_utils import datetime_naive_to_local, datetime_to_new_tz
-from .utils import assert_macos, noop
+from .platform import assert_macos
+from .utils import noop
 
 assert_macos()
 

osxphotos/export_db.py

@@ -27,7 +27,7 @@ import osxphotos
 from ._constants import OSXPHOTOS_EXPORT_DB, SQLITE_CHECK_SAME_THREAD
 from ._version import __version__
 from .fileutil import FileUtil
-from .utils import normalize_fs_path
+from .unicode import normalize_fs_path
 
 __all__ = [
     "ExportDB",

osxphotos/fileutil.py

@@ -10,7 +10,8 @@ from abc import ABC, abstractmethod
 from tempfile import TemporaryDirectory
 
 from .imageconverter import ImageConverter
-from .utils import is_macos, normalize_fs_path
+from .platform import is_macos
+from .unicode import normalize_fs_path
 
 if is_macos:
     import Foundation

osxphotos/imageconverter.py

@@ -10,7 +10,7 @@ import sys
 # needed to capture system-level stderr
 from wurlitzer import pipes
 
-from .utils import is_macos
+from .platform import is_macos
 
 if is_macos:
     import Metal

osxphotos/path_utils.py

@@ -13,7 +13,7 @@ filename is needed.
 
 import pathvalidate
 
-from osxphotos.utils import normalize_unicode
+from osxphotos.unicode import normalize_unicode
 
 from ._constants import MAX_DIRNAME_LEN, MAX_FILENAME_LEN
 

osxphotos/photoexporter.py

@@ -36,17 +36,17 @@ from .exiftool import ExifTool, ExifToolCaching, exiftool_can_write, get_exiftoo
 from .export_db import ExportDB, ExportDBTemp
 from .fileutil import FileUtil
 from .phototemplate import RenderOptions
+from .platform import is_macos
 from .rich_utils import add_rich_markup_tag
+from .unicode import normalize_fs_path
 from .uti import get_preferred_uti_extension
 from .utils import (
     hexdigest,
     increment_filename,
     increment_filename_with_count,
-    is_macos,
     lineno,
     list_directory,
     lock_filename,
-    normalize_fs_path,
     unlock_filename,
 )
 

osxphotos/photoinfo.py

@@ -61,11 +61,12 @@ from .photoexporter import ExportOptions, PhotoExporter
 from .phototables import PhotoTables
 from .phototemplate import PhotoTemplate, RenderOptions
 from .placeinfo import PlaceInfo4, PlaceInfo5
+from .platform import assert_macos, is_macos
 from .query_builder import get_query
 from .scoreinfo import ScoreInfo
 from .searchinfo import SearchInfo
 from .uti import get_preferred_uti_extension, get_uti_for_extension
-from .utils import _get_resource_loc, assert_macos, hexdigest, is_macos, list_directory
+from .utils import _get_resource_loc, hexdigest, list_directory
 
 if is_macos:
     from osxmetadata import OSXMetaData

osxphotos/photokit.py

@@ -23,6 +23,8 @@ import sys
 import threading
 import time
 
+from .platform import get_macos_version
+
 assert sys.platform == "darwin"
 
 import AVFoundation
@@ -37,7 +39,7 @@ from wurlitzer import pipes
 
 from .fileutil import FileUtil
 from .uti import get_preferred_uti_extension
-from .utils import get_macos_version, increment_filename
+from .utils import increment_filename
 
 __all__ = [
     "NSURL_to_path",

osxphotos/photosalbum.py

@@ -1,11 +1,15 @@
 """ PhotosAlbum class to create an album in default Photos library and add photos to it """
 
+from __future__ import annotations
+
+import unicodedata
 from typing import List, Optional
 
 from more_itertools import chunked
 
 from .photoinfo import PhotoInfo
-from .utils import assert_macos, noop, pluralize
+from .platform import assert_macos
+from .utils import noop, pluralize
 
 assert_macos()
 
@@ -15,19 +19,36 @@ from photoscript import Album, Folder, Photo, PhotosLibrary
 __all__ = ["PhotosAlbum", "PhotosAlbumPhotoScript"]
 
 
+def get_unicode_variants(s: str) -> list[str]:
+    """Get all unicode variants of string"""
+    variants = []
+    for form in ["NFC", "NFD", "NFKC", "NFKD"]:
+        normalized = unicodedata.normalize(form, s)
+        variants.append(normalized)
+    return variants
+
+
 def folder_by_path(folders: List[str], verbose: Optional[callable] = None) -> Folder:
     """Get (and create if necessary) a Photos Folder by path (passed as list of folder names)"""
     library = PhotosLibrary()
     verbose = verbose or noop
     top_folder_name = folders.pop(0)
-    top_folder = library.folder(top_folder_name, top_level=True)
-    if not top_folder:
+
+    for folder_variant in get_unicode_variants(top_folder_name):
+        top_folder = library.folder(folder_variant, top_level=True)
+        if top_folder is not None:
+            break
+    else:
         verbose(f"Creating folder '{top_folder_name}'")
         top_folder = library.create_folder(top_folder_name)
+
     current_folder = top_folder
     for folder_name in folders:
-        folder = current_folder.folder(folder_name)
-        if not folder:
+        for folder_variant in get_unicode_variants(folder_name):
+            folder = current_folder.folder(folder_variant)
+            if folder is not None:
+                break
+        else:
             verbose(f"Creating folder '{folder_name}'")
             folder = current_folder.create_folder(folder_name)
         current_folder = folder
@@ -44,15 +65,24 @@ def album_by_path(
         # have folders
         album_name = folders_album.pop()
         folder = folder_by_path(folders_album, verbose)
-        album = folder.album(album_name)
-        if album is None:
+        for album_variant in get_unicode_variants(album_name):
+            # Get album if it exists
+            # need to check every unicode variant to avoid creating duplicate albums with same visual representation (#1085)
+            album = folder.album(album_variant)
+            if album is not None:
+                break
+        else:
             verbose(f"Creating album '{album_name}'")
             album = folder.create_album(album_name)
     else:
         # only have album name
         album_name = folders_album[0]
-        album = library.album(album_name, top_level=True)
-        if album is None:
+        for album_variant in get_unicode_variants(album_name):
+            album = library.album(album_variant, top_level=True)
+            if album is not None:
+                break
+        else:
+            # album doesn't exist, create it
             verbose(f"Creating album '{album_name}'")
             album = library.create_album(album_name)
 
@@ -101,15 +131,10 @@ class PhotosAlbum:
             try:
                 photos.append(photoscript.Photo(p.uuid))
             except Exception as e:
-                print(
-                    f"Error creating Photo object for photo {self._format_uuid(p.uuid)}: {e}"
-                )
                 self.verbose(
                     f"Error creating Photo object for photo {self._format_uuid(p.uuid)}: {e}"
                 )
-        print(f"photos: {photos}")
         for photolist in chunked(photos, 10):
-            print(f"photolist: {photolist}")
             self.album.add(photolist)
         photo_len = len(photo_list)
         self.verbose(

osxphotos/photoscript_utils.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import sqlite3
 
-from .utils import assert_macos
+from .platform import assert_macos
 
 assert_macos()
 

osxphotos/photosdb/_photosdb_process_comments.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 
 from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION, TIME_DELTA
 from ..sqlite_utils import sqlite_open_ro
-from ..utils import normalize_unicode
+from ..unicode import normalize_unicode
 
 
 def _process_comments(self):

osxphotos/photosdb/_photosdb_process_faceinfo.py

@@ -4,7 +4,7 @@
 
 from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
 from ..sqlite_utils import sqlite_open_ro
-from ..utils import normalize_unicode
+from ..unicode import normalize_unicode
 from .photosdb_utils import get_db_version
 
 """

osxphotos/photosdb/_photosdb_process_searchinfo.py

@@ -10,7 +10,7 @@ from functools import lru_cache
 
 from .._constants import _PHOTOS_4_VERSION, search_category_factory
 from ..sqlite_utils import sqlite_db_is_locked, sqlite_open_ro
-from ..utils import normalize_unicode
+from ..unicode import normalize_unicode
 
 """
     This module should be imported in the class defintion of PhotosDB in photosdb.py

osxphotos/photosdb/photosdb.py

@@ -56,17 +56,12 @@ from ..fileutil import FileUtil
 from ..personinfo import PersonInfo
 from ..photoinfo import PhotoInfo
 from ..phototemplate import RenderOptions
+from ..platform import get_macos_version, is_macos
 from ..queryoptions import QueryOptions
 from ..rich_utils import add_rich_markup_tag
 from ..sqlite_utils import sqlite_db_is_locked, sqlite_open_ro
-from ..utils import (
-    _check_file_exists,
-    get_last_library_path,
-    get_macos_version,
-    is_macos,
-    noop,
-    normalize_unicode,
-)
+from ..unicode import normalize_unicode
+from ..utils import _check_file_exists, get_last_library_path, noop
 from .photosdb_utils import get_db_model_version, get_db_version
 
 if is_macos:

osxphotos/placeinfo.py

@@ -11,8 +11,7 @@ from collections import namedtuple  # pylint: disable=syntax-error
 import yaml
 from bpylist2 import archiver
 
-from ._constants import UNICODE_FORMAT
-from .utils import normalize_unicode
+from .unicode import normalize_unicode
 
 __all__ = [
     "PLRevGeoLocationInfo",

osxphotos/platform.py

@@ -0,0 +1,29 @@
+"""Functions for multi-platform support"""
+
+import platform
+import sys
+
+is_macos = sys.platform == "darwin"
+
+
+def assert_macos():
+    assert is_macos, "This feature only runs on macOS"
+
+
+def get_macos_version():
+    assert_macos()
+    # returns tuple of str containing OS version
+    # e.g. 10.13.6 = ("10", "13", "6")
+    version = platform.mac_ver()[0].split(".")
+    if len(version) == 2:
+        (ver, major) = version
+        minor = "0"
+    elif len(version) == 3:
+        (ver, major, minor) = version
+    else:
+        raise (
+            ValueError(
+                f"Could not parse version string: {platform.mac_ver()} {version}"
+            )
+        )
+    return (ver, major, minor)

osxphotos/text_detection.py

@@ -4,7 +4,7 @@ import logging
 import sys
 from typing import List, Optional
 
-from .utils import assert_macos, get_macos_version
+from .platform import assert_macos, get_macos_version
 
 assert_macos()
 

osxphotos/timezones.py

@@ -2,7 +2,7 @@
 
 from typing import Union
 
-from .utils import is_macos
+from .platform import is_macos
 
 
 def format_offset_time(offset: int) -> str:

osxphotos/unicode.py

@@ -0,0 +1,90 @@
+"""Utilities for working with unicode strings."""
+
+from __future__ import annotations
+
+import pathlib
+import unicodedata
+from typing import Literal, TypeVar, Union
+
+from osxphotos.platform import is_macos
+
+# Unicode format to use for comparing strings
+DEFAULT_UNICODE_FORM = "NFC"
+
+# global unicode format
+_GLOBAL_UNICODE_FORM = DEFAULT_UNICODE_FORM
+
+# global unicode format to use for filesystem paths
+_GLOBAL_UNICODE_FS_FORM = "NFD" if is_macos else "NFC"
+
+PathType = TypeVar("PathType", bound=Union[str, pathlib.Path])
+
+UnicodeDataType = TypeVar(
+    "UnicodeDataType", bound=Union[str, list[str], tuple[str, ...], None]
+)
+
+__all__ = [
+    "get_unicode_form",
+    "set_unicode_form",
+    "get_unicode_fs_form",
+    "set_unicode_fs_form",
+    "normalize_fs_path",
+    "normalize_unicode",
+    "DEFAULT_UNICODE_FORM",
+]
+
+
+def get_unicode_form() -> Literal["NFC", "NFKC", "NFD", "NFKD"]:
+    """Return the global unicode format"""
+    global _GLOBAL_UNICODE_FORM
+    return _GLOBAL_UNICODE_FORM
+
+
+def set_unicode_form(format: Literal["NFC", "NFKC", "NFD", "NFKD"]) -> None:
+    """Set the global unicode format"""
+
+    if format not in ["NFC", "NFKC", "NFD", "NFKD"]:
+        raise ValueError(f"Invalid unicode format: {format}")
+
+    global _GLOBAL_UNICODE_FORM
+    _GLOBAL_UNICODE_FORM = format
+
+
+def get_unicode_fs_form() -> Literal["NFC", "NFKC", "NFD", "NFKD"]:
+    """Return the global unicode filesystem format"""
+    global _GLOBAL_UNICODE_FS_FORM
+    return _GLOBAL_UNICODE_FS_FORM
+
+
+def set_unicode_fs_form(format: str) -> Literal["NFC", "NFKC", "NFD", "NFKD"]:
+    """Set the global unicode filesystem format"""
+
+    if format not in ["NFC", "NFKC", "NFD", "NFKD"]:
+        raise ValueError(f"Invalid unicode format: {format}")
+
+    global _GLOBAL_UNICODE_FS_FORM
+    _GLOBAL_UNICODE_FS_FORM = format
+
+
+def normalize_fs_path(path: PathType) -> PathType:
+    """Normalize filesystem paths with unicode in them"""
+    form = get_unicode_fs_form()
+    if isinstance(path, pathlib.Path):
+        return pathlib.Path(unicodedata.normalize(form, str(path)))
+    else:
+        return unicodedata.normalize(form, path)
+
+
+def normalize_unicode(value: UnicodeDataType) -> UnicodeDataType:
+    """normalize unicode data"""
+    form = get_unicode_form()
+    if value is None:
+        return None
+    if isinstance(value, tuple):
+        return tuple(unicodedata.normalize(form, v) for v in value)
+    elif isinstance(value, list):
+        return [unicodedata.normalize(form, v) for v in value]
+    elif isinstance(value, str):
+        return unicodedata.normalize(form, value)
+    else:
+        return value

osxphotos/uti.py

@@ -24,7 +24,7 @@ import subprocess
 import sys
 import tempfile
 
-from .utils import assert_macos, get_macos_version, is_macos
+from .platform import assert_macos, get_macos_version, is_macos
 
 if is_macos:
     import CoreServices

osxphotos/utils.py

@@ -1,5 +1,7 @@
 """ Utility functions used in osxphotos """
 
+from __future__ import annotations
+
 import datetime
 import fnmatch
 import hashlib
@@ -9,30 +11,29 @@ import logging
 import os
 import os.path
 import pathlib
-import platform
 import re
 import subprocess
 import sys
-import unicodedata
 import urllib.parse
 from plistlib import load as plistload
-from typing import Any, Callable, List, Optional, Tuple, TypeVar, Union
+from typing import Callable, List, Optional, Tuple, TypeVar, Union
 from uuid import UUID
 
 import requests
 import shortuuid
 
-from ._constants import UNICODE_FORMAT
+from osxphotos.platform import get_macos_version, is_macos
+from osxphotos.unicode import normalize_fs_path
+
+T = TypeVar("T", bound=Union[str, pathlib.Path])
 
 logger = logging.getLogger("osxphotos")
 
+
 __all__ = [
-    "is_macos",
-    "assert_macos",
     "dd_to_dms_str",
     "expand_and_validate_filepath",
     "get_last_library_path",
-    "get_macos_version",
     "get_system_library_path",
     "hexdigest",
     "increment_filename_with_count",
@@ -43,8 +44,6 @@ __all__ = [
     "load_function",
     "lock_filename",
     "noop",
-    "normalize_fs_path",
-    "normalize_unicode",
     "pluralize",
     "shortuuid_to_uuid",
     "uuid_to_shortuuid",
@@ -54,13 +53,6 @@ __all__ = [
 VERSION_INFO_URL = "https://pypi.org/pypi/osxphotos/json"
 
 
-is_macos = sys.platform == "darwin"
-
-
-def assert_macos():
-    assert is_macos, "This feature only runs on macOS"
-
-
 if is_macos:
     import CoreFoundation
 
@@ -78,25 +70,6 @@ def lineno(filename):
     return f"{filename}: {line}"
 
 
-def get_macos_version():
-    assert_macos()
-    # returns tuple of str containing OS version
-    # e.g. 10.13.6 = ("10", "13", "6")
-    version = platform.mac_ver()[0].split(".")
-    if len(version) == 2:
-        (ver, major) = version
-        minor = "0"
-    elif len(version) == 3:
-        (ver, major, minor) = version
-    else:
-        raise (
-            ValueError(
-                f"Could not parse version string: {platform.mac_ver()} {version}"
-            )
-        )
-    return (ver, major, minor)
-
-
 def _check_file_exists(filename):
     """returns true if file exists and is not a directory
     otherwise returns false"""
@@ -280,16 +253,6 @@ def list_photo_libraries():
     return lib_list
 
 
-T = TypeVar("T", bound=Union[str, pathlib.Path])
-
-
-def normalize_fs_path(path: T) -> T:
-    """Normalize filesystem paths with unicode in them"""
-    form = "NFD" if is_macos else "NFC"
-    if isinstance(path, pathlib.Path):
-        return pathlib.Path(unicodedata.normalize(form, str(path)))
-    else:
-        return unicodedata.normalize(form, path)
 
 
 # def findfiles(pattern, path):
@@ -378,18 +341,6 @@ def list_directory(
     return files
 
 
-def normalize_unicode(value) -> Any:
-    """normalize unicode data"""
-    if value is None:
-        return None
-    if isinstance(value, (tuple, list)):
-        return tuple(unicodedata.normalize(UNICODE_FORMAT, v) for v in value)
-    elif isinstance(value, str):
-        return unicodedata.normalize(UNICODE_FORMAT, value)
-    else:
-        return value
-
-
 def increment_filename_with_count(
     filepath: Union[str, pathlib.Path], count: int = 0, lock: bool = False
 ) -> Tuple[str, int]:

tests/README.md

@@ -23,7 +23,7 @@ One test for locale does not run on GitHub's automated workflow and will look fo
 
 A couple of tests require interaction with Photos and configuring a specific test library. Currently these run only on Catalina.  The tests must be specified by using a pytest flag. Only one of these interactive tests can be run at a time.  The current flags are:
 
---addalbum: test --add-to-album options
+--addalbum: test --add-to-album options (pytest -vv tests/test_photosalbum_unicode.py tests/test_cli_add_to_album.py --addalbum)
 --timewarp: test `osxphotos timewarp`
 --test-import: test `osxphotos import`
 --test-sync: test `osxphotos sync`

tests/conftest.py

@@ -7,7 +7,7 @@ import time
 
 import pytest
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_catalina_10_15_7.py

@@ -15,7 +15,7 @@ import pytest
 import osxphotos
 from osxphotos._constants import _UNKNOWN_PERSON
 from osxphotos.photoexporter import PhotoExporter
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "15"

tests/test_cli.py

@@ -36,7 +36,9 @@ from osxphotos.cli import (
 )
 from osxphotos.exiftool import ExifTool, get_exiftool_path
 from osxphotos.fileutil import FileUtil
-from osxphotos.utils import is_macos, noop, normalize_fs_path, normalize_unicode
+from osxphotos.platform import is_macos
+from osxphotos.unicode import normalize_fs_path, normalize_unicode
+from osxphotos.utils import noop
 
 if is_macos:
     from osxmetadata import OSXMetaData, Tag

tests/test_cli_add_locations.py

@@ -3,7 +3,7 @@
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_add_to_album.py

@@ -5,7 +5,7 @@ import os
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_all_commands.py

@@ -44,7 +44,7 @@ from osxphotos.cli import (
     tutorial,
     version,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     from osxphotos.cli import uuid

tests/test_cli_batch_edit.py

@@ -9,7 +9,7 @@ import pytest
 from click.testing import CliRunner
 
 import osxphotos
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_import.py

@@ -9,6 +9,7 @@ import re
 import shutil
 import sqlite3
 import time
+import unicodedata
 from datetime import datetime
 from tempfile import TemporaryDirectory
 from typing import Dict
@@ -21,7 +22,7 @@ from osxphotos import PhotosDB, QueryOptions
 from osxphotos._constants import UUID_PATTERN
 from osxphotos.datetime_utils import datetime_remove_tz
 from osxphotos.exiftool import get_exiftool_path
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 from tests.conftest import get_os_version
 
 if is_macos:
@@ -44,17 +45,17 @@ TEST_DATA = {
     TEST_IMAGE_1: {
         "title": "Waves crashing on rocks",
         "description": "Used for testing osxphotos",
-        "keywords": ["osxphotos"],
+        "keywords": ["osxphotos", "Sümmer"],
         "lat": 33.7150638888889,
         "lon": -118.319672222222,
         "check_templates": [
             "exiftool title: Waves crashing on rocks",
             "exiftool description: Used for testing osxphotos",
-            "exiftool keywords: ['osxphotos']",
+            "exiftool keywords: ['osxphotos', 'Sümmer']",
             "exiftool location: (33.7150638888889, -118.319672222222)",
             "title: {exiftool:XMP:Title}: Waves crashing on rocks",
             "description: {exiftool:IPTC:Caption-Abstract}: Used for testing osxphotos",
-            "keyword: {exiftool:IPTC:Keywords}: ['osxphotos']",
+            "keyword: {exiftool:IPTC:Keywords}: ['osxphotos', 'Sümmer']",
             "album: {filepath.parent}: test-images",
         ],
     },
@@ -536,7 +537,44 @@ def test_import_keyword_merge():
     photo_1 = Photo(uuid_1)
 
     assert photo_1.filename == file_1
-    assert sorted(photo_1.keywords) == ["Bar", "Foo", "osxphotos"]
+    assert sorted(photo_1.keywords) == sorted(list(set(["Bar", "Foo"] + TEST_DATA[TEST_IMAGE_1]["keywords"])))
+
+@pytest.mark.skipif(exiftool_path is None, reason="exiftool not installed")
+@pytest.mark.test_import
+def test_import_keyword_merge_unicode():
+    """Test import with --keyword and --merge-keywords with unicode keywords (#1085)"""
+    cwd = os.getcwd()
+    test_image_1 = os.path.join(cwd, TEST_IMAGE_1)
+    runner = CliRunner()
+    result = runner.invoke(
+        import_cli,
+        [
+            "--verbose",
+            "--clear-metadata",
+            "--exiftool",
+            "--keyword",
+            "Bar",
+            "--keyword",
+            "Foo",
+            "--keyword",
+            unicodedata.normalize("NFD", "Sümmer"),
+            "--keyword",
+            unicodedata.normalize("NFC", "Sümmer"),
+            "--merge-keywords",
+            test_image_1,
+        ],
+        terminal_width=TERMINAL_WIDTH,
+    )
+
+    assert result.exit_code == 0
+
+    import_data = parse_import_output(result.output)
+    file_1 = pathlib.Path(test_image_1).name
+    uuid_1 = import_data[file_1]
+    photo_1 = Photo(uuid_1)
+
+    assert photo_1.filename == file_1
+    assert sorted(photo_1.keywords) == sorted(list(set(["Bar", "Foo"] + TEST_DATA[TEST_IMAGE_1]["keywords"])))
 
 
 @pytest.mark.test_import

tests/test_cli_sync.py

@@ -6,7 +6,7 @@ import os
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_export_catalina_10_15_7_use_photos_export.py

@@ -3,7 +3,7 @@ import os
 import pytest
 
 from osxphotos._constants import _UNKNOWN_PERSON
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "15"

tests/test_monterey_dev_beta_12_0_0.py

@@ -14,7 +14,7 @@ import pytest
 import osxphotos
 from osxphotos._constants import _UNKNOWN_PERSON
 from osxphotos.photoexporter import PhotoExporter
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 # SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "17"

tests/test_photokit.py

@@ -6,7 +6,7 @@ import tempfile
 
 import pytest
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     from osxphotos.photokit import (

tests/test_photosalbum_unicode.py

@@ -0,0 +1,90 @@
+"""Test unicode names in PhotoAlbum PhotoAlbumPhotoScript (#1085)"""
+
+import pathlib
+from unicodedata import normalize
+
+import pytest
+
+from osxphotos.platform import is_macos
+
+if not is_macos:
+    pytest.skip("requires macOS", allow_module_level=True)
+
+import osxphotos
+from osxphotos.photosalbum import PhotosAlbum, PhotosAlbumPhotoScript
+from osxphotos.unicode import *
+
+UNICODE_FOLDER_NFC = normalize("NFC", "FolderUnicode/føldêr2")
+UNICODE_FOLDER_NFD = normalize("NFD", UNICODE_FOLDER_NFC)
+
+UNICODE_ALBUM_NFC = normalize("NFC", "âlbüm")
+UNICODE_ALBUM_NFD = normalize("NFD", UNICODE_ALBUM_NFC)
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_album(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created"""
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = UNICODE_ALBUM_NFC
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder=None)
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = UNICODE_ALBUM_NFD
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder=None)
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_folder_album_1(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created when album is in a folder"""
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFC}"
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder="/")
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = f"{UNICODE_FOLDER_NFD}/{UNICODE_ALBUM_NFD}"
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder="/")
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_folder_album_2(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created when album is in a folder
+
+    This is a variation of test_unicode_folder_album_1 where the album is created in the same unicode folder as the previous album
+    """
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFC}"
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder="/")
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFD}"
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder="/")
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid

tests/test_template.py

@@ -15,7 +15,7 @@ from osxphotos.phototemplate import (
     PhotoTemplate,
     RenderOptions,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 from .locale_util import setlocale
 from .photoinfo_mock import PhotoInfoMock

tests/test_unicode.py

@@ -0,0 +1,110 @@
+"""Test unicode utilities"""
+
+import pathlib
+from unicodedata import normalize
+
+import pytest
+
+from osxphotos.platform import is_macos
+from osxphotos.unicode import *
+
+UNICODE_PATH_NFC = normalize("NFC", "/path/to/ünicøde")
+UNICODE_PATH_NFD = normalize("NFD", UNICODE_PATH_NFC)
+
+UNICODE_STR_NFC = normalize("NFC", "âbc")
+UNICODE_STR_NFD = normalize("NFD", UNICODE_STR_NFC)
+
+UNICODE_LIST_NFC = [normalize("NFC", "âbc"), normalize("NFC", "dê")]
+UNICODE_LIST_NFD = [normalize("NFD", "âbc"), normalize("NFD", "dê")]
+
+
+def test_get_unicode_format():
+    set_unicode_form("NFC")
+    assert get_unicode_form() == "NFC"
+
+
+def test_set_unicode_format():
+    set_unicode_form("NFD")
+    assert get_unicode_form() == "NFD"
+
+    set_unicode_form("NFC")
+    assert get_unicode_form() == "NFC"
+
+    # test invalid format
+    with pytest.raises(ValueError):
+        set_unicode_form("foo")
+
+    # Reset to correct format based
+    set_unicode_form(DEFAULT_UNICODE_FORM)
+
+
+def test_set_unicode_fs_format():
+    set_unicode_fs_form("NFC")
+    assert get_unicode_fs_form() == "NFC"
+
+    set_unicode_fs_form("NFD")
+    assert get_unicode_fs_form() == "NFD"
+
+    # test invalid format
+    with pytest.raises(ValueError):
+        set_unicode_fs_form("foo")
+
+    # Reset to correct format based on platform
+    set_unicode_fs_form("NFD" if is_macos else "NFC")
+
+
+def test_normalize_fs_path():
+    # Test with string path in NFC format
+    set_unicode_fs_form("NFC")
+    assert normalize_fs_path(UNICODE_PATH_NFD) == UNICODE_PATH_NFC
+
+    # Test with string path in NFD format
+    set_unicode_fs_form("NFD")
+    assert normalize_fs_path(UNICODE_PATH_NFC) == UNICODE_PATH_NFD
+
+    # Test with pathlib.Path object in NFC format
+    set_unicode_fs_form("NFC")
+    assert normalize_fs_path(pathlib.Path(UNICODE_PATH_NFD)) == pathlib.Path(
+        UNICODE_PATH_NFC
+    )
+
+    # Test with pathlib.Path object in NFD format
+    set_unicode_fs_form("NFD")
+    assert normalize_fs_path(pathlib.Path(UNICODE_PATH_NFC)) == pathlib.Path(
+        UNICODE_PATH_NFD
+    )
+
+    # Reset to correct format based on platform
+    set_unicode_fs_form("NFD" if is_macos else "NFC")
+
+
+def test_normalize_unicode():
+    # Test with str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(UNICODE_STR_NFD) == UNICODE_STR_NFC
+
+    # Test with str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(UNICODE_STR_NFC) == UNICODE_STR_NFD
+
+    # Test with list of str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(UNICODE_LIST_NFD) == UNICODE_LIST_NFC
+
+    # Test with list of str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(UNICODE_LIST_NFC) == UNICODE_LIST_NFD
+
+    # Test with tuple of str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(tuple(UNICODE_LIST_NFD)) == tuple(UNICODE_LIST_NFC)
+
+    # Test with tuple of str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(tuple(UNICODE_LIST_NFC)) == tuple(UNICODE_LIST_NFD)
+
+    # Test with None
+    assert normalize_unicode(None) is None
+
+    # Reset to correct format based
+    set_unicode_form(DEFAULT_UNICODE_FORM)

tests/test_uti.py

@@ -3,12 +3,12 @@
 import pytest
 
 import osxphotos.uti
+from osxphotos.platform import is_macos
 from osxphotos.uti import (
     _get_uti_from_mdls,
     get_preferred_uti_extension,
     get_uti_for_extension,
 )
-from osxphotos.utils import is_macos
 
 EXT_DICT = {"heic": "public.heic", "jpg": "public.jpeg", ".jpg": "public.jpeg"}
 UTI_DICT = {"public.heic": "heic", "public.jpeg": "jpeg"}