Unicode refactor (#1101)

* Began refactoring for improving unicode handling

* Added platform and unicode modules

* Added tests for unicode utilities

* Added tests for unicode utilities

* Added tests for unicode utilities

* Added tests for unicode utilities

* Fixed unicode tests for linux

* Fixed unicode tests for linux

* Fixed duplicate alubm name with --add-to-album

* Fixed test for linux

* Fix for duplicate unicode kewyords, see #907, #1085

tests/README.md

@@ -23,7 +23,7 @@ One test for locale does not run on GitHub's automated workflow and will look fo
 
 A couple of tests require interaction with Photos and configuring a specific test library. Currently these run only on Catalina.  The tests must be specified by using a pytest flag. Only one of these interactive tests can be run at a time.  The current flags are:
 
---addalbum: test --add-to-album options
+--addalbum: test --add-to-album options (pytest -vv tests/test_photosalbum_unicode.py tests/test_cli_add_to_album.py --addalbum)
 --timewarp: test `osxphotos timewarp`
 --test-import: test `osxphotos import`
 --test-sync: test `osxphotos sync`

tests/conftest.py

@@ -7,7 +7,7 @@ import time
 
 import pytest
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_catalina_10_15_7.py

@@ -15,7 +15,7 @@ import pytest
 import osxphotos
 from osxphotos._constants import _UNKNOWN_PERSON
 from osxphotos.photoexporter import PhotoExporter
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "15"

tests/test_cli.py

@@ -36,7 +36,9 @@ from osxphotos.cli import (
 )
 from osxphotos.exiftool import ExifTool, get_exiftool_path
 from osxphotos.fileutil import FileUtil
-from osxphotos.utils import is_macos, noop, normalize_fs_path, normalize_unicode
+from osxphotos.platform import is_macos
+from osxphotos.unicode import normalize_fs_path, normalize_unicode
+from osxphotos.utils import noop
 
 if is_macos:
     from osxmetadata import OSXMetaData, Tag

tests/test_cli_add_locations.py

@@ -3,7 +3,7 @@
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_add_to_album.py

@@ -5,7 +5,7 @@ import os
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_all_commands.py

@@ -44,7 +44,7 @@ from osxphotos.cli import (
     tutorial,
     version,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     from osxphotos.cli import uuid

tests/test_cli_batch_edit.py

@@ -9,7 +9,7 @@ import pytest
 from click.testing import CliRunner
 
 import osxphotos
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_cli_import.py

@@ -9,6 +9,7 @@ import re
 import shutil
 import sqlite3
 import time
+import unicodedata
 from datetime import datetime
 from tempfile import TemporaryDirectory
 from typing import Dict
@@ -21,7 +22,7 @@ from osxphotos import PhotosDB, QueryOptions
 from osxphotos._constants import UUID_PATTERN
 from osxphotos.datetime_utils import datetime_remove_tz
 from osxphotos.exiftool import get_exiftool_path
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 from tests.conftest import get_os_version
 
 if is_macos:
@@ -44,17 +45,17 @@ TEST_DATA = {
     TEST_IMAGE_1: {
         "title": "Waves crashing on rocks",
         "description": "Used for testing osxphotos",
-        "keywords": ["osxphotos"],
+        "keywords": ["osxphotos", "Sümmer"],
         "lat": 33.7150638888889,
         "lon": -118.319672222222,
         "check_templates": [
             "exiftool title: Waves crashing on rocks",
             "exiftool description: Used for testing osxphotos",
-            "exiftool keywords: ['osxphotos']",
+            "exiftool keywords: ['osxphotos', 'Sümmer']",
             "exiftool location: (33.7150638888889, -118.319672222222)",
             "title: {exiftool:XMP:Title}: Waves crashing on rocks",
             "description: {exiftool:IPTC:Caption-Abstract}: Used for testing osxphotos",
-            "keyword: {exiftool:IPTC:Keywords}: ['osxphotos']",
+            "keyword: {exiftool:IPTC:Keywords}: ['osxphotos', 'Sümmer']",
             "album: {filepath.parent}: test-images",
         ],
     },
@@ -536,7 +537,44 @@ def test_import_keyword_merge():
     photo_1 = Photo(uuid_1)
 
     assert photo_1.filename == file_1
-    assert sorted(photo_1.keywords) == ["Bar", "Foo", "osxphotos"]
+    assert sorted(photo_1.keywords) == sorted(list(set(["Bar", "Foo"] + TEST_DATA[TEST_IMAGE_1]["keywords"])))
+
+@pytest.mark.skipif(exiftool_path is None, reason="exiftool not installed")
+@pytest.mark.test_import
+def test_import_keyword_merge_unicode():
+    """Test import with --keyword and --merge-keywords with unicode keywords (#1085)"""
+    cwd = os.getcwd()
+    test_image_1 = os.path.join(cwd, TEST_IMAGE_1)
+    runner = CliRunner()
+    result = runner.invoke(
+        import_cli,
+        [
+            "--verbose",
+            "--clear-metadata",
+            "--exiftool",
+            "--keyword",
+            "Bar",
+            "--keyword",
+            "Foo",
+            "--keyword",
+            unicodedata.normalize("NFD", "Sümmer"),
+            "--keyword",
+            unicodedata.normalize("NFC", "Sümmer"),
+            "--merge-keywords",
+            test_image_1,
+        ],
+        terminal_width=TERMINAL_WIDTH,
+    )
+
+    assert result.exit_code == 0
+
+    import_data = parse_import_output(result.output)
+    file_1 = pathlib.Path(test_image_1).name
+    uuid_1 = import_data[file_1]
+    photo_1 = Photo(uuid_1)
+
+    assert photo_1.filename == file_1
+    assert sorted(photo_1.keywords) == sorted(list(set(["Bar", "Foo"] + TEST_DATA[TEST_IMAGE_1]["keywords"])))
 
 
 @pytest.mark.test_import

tests/test_cli_sync.py

@@ -6,7 +6,7 @@ import os
 import pytest
 from click.testing import CliRunner
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     import photoscript

tests/test_export_catalina_10_15_7_use_photos_export.py

@@ -3,7 +3,7 @@ import os
 import pytest
 
 from osxphotos._constants import _UNKNOWN_PERSON
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "15"

tests/test_monterey_dev_beta_12_0_0.py

@@ -14,7 +14,7 @@ import pytest
 import osxphotos
 from osxphotos._constants import _UNKNOWN_PERSON
 from osxphotos.photoexporter import PhotoExporter
-from osxphotos.utils import get_macos_version, is_macos
+from osxphotos.platform import get_macos_version, is_macos
 
 OS_VERSION = get_macos_version() if is_macos else (None, None, None)
 # SKIP_TEST = "OSXPHOTOS_TEST_EXPORT" not in os.environ or OS_VERSION[1] != "17"

tests/test_photokit.py

@@ -6,7 +6,7 @@ import tempfile
 
 import pytest
 
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 if is_macos:
     from osxphotos.photokit import (

tests/test_photosalbum_unicode.py

@@ -0,0 +1,90 @@
+"""Test unicode names in PhotoAlbum PhotoAlbumPhotoScript (#1085)"""
+
+import pathlib
+from unicodedata import normalize
+
+import pytest
+
+from osxphotos.platform import is_macos
+
+if not is_macos:
+    pytest.skip("requires macOS", allow_module_level=True)
+
+import osxphotos
+from osxphotos.photosalbum import PhotosAlbum, PhotosAlbumPhotoScript
+from osxphotos.unicode import *
+
+UNICODE_FOLDER_NFC = normalize("NFC", "FolderUnicode/føldêr2")
+UNICODE_FOLDER_NFD = normalize("NFD", UNICODE_FOLDER_NFC)
+
+UNICODE_ALBUM_NFC = normalize("NFC", "âlbüm")
+UNICODE_ALBUM_NFD = normalize("NFD", UNICODE_ALBUM_NFC)
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_album(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created"""
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = UNICODE_ALBUM_NFC
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder=None)
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = UNICODE_ALBUM_NFD
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder=None)
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_folder_album_1(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created when album is in a folder"""
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFC}"
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder="/")
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = f"{UNICODE_FOLDER_NFD}/{UNICODE_ALBUM_NFD}"
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder="/")
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid
+
+
+@pytest.mark.skipif(not is_macos, reason="requires macOS")
+@pytest.mark.addalbum
+def test_unicode_folder_album_2(addalbum_library):
+    """Test that unicode album name is handled correctly and a duplicate album is not created when album is in a folder
+
+    This is a variation of test_unicode_folder_album_1 where the album is created in the same unicode folder as the previous album
+    """
+
+    # get some photos to add
+    photosdb = osxphotos.PhotosDB()
+    photos = photosdb.query(osxphotos.QueryOptions(person=["Katie"]))
+
+    # get the album
+    album_name_nfc = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFC}"
+    album_nfc = PhotosAlbum(album_name_nfc, split_folder="/")
+    album_nfc.add_list(photos)
+
+    # again with NFD
+    album_name_nfd = f"{UNICODE_FOLDER_NFC}/{UNICODE_ALBUM_NFD}"
+    album_nfd = PhotosAlbum(album_name_nfd, split_folder="/")
+    album_nfd.add_list(photos)
+
+    assert album_nfc.album.uuid == album_nfd.album.uuid

tests/test_template.py

@@ -15,7 +15,7 @@ from osxphotos.phototemplate import (
     PhotoTemplate,
     RenderOptions,
 )
-from osxphotos.utils import is_macos
+from osxphotos.platform import is_macos
 
 from .locale_util import setlocale
 from .photoinfo_mock import PhotoInfoMock

tests/test_unicode.py

@@ -0,0 +1,110 @@
+"""Test unicode utilities"""
+
+import pathlib
+from unicodedata import normalize
+
+import pytest
+
+from osxphotos.platform import is_macos
+from osxphotos.unicode import *
+
+UNICODE_PATH_NFC = normalize("NFC", "/path/to/ünicøde")
+UNICODE_PATH_NFD = normalize("NFD", UNICODE_PATH_NFC)
+
+UNICODE_STR_NFC = normalize("NFC", "âbc")
+UNICODE_STR_NFD = normalize("NFD", UNICODE_STR_NFC)
+
+UNICODE_LIST_NFC = [normalize("NFC", "âbc"), normalize("NFC", "dê")]
+UNICODE_LIST_NFD = [normalize("NFD", "âbc"), normalize("NFD", "dê")]
+
+
+def test_get_unicode_format():
+    set_unicode_form("NFC")
+    assert get_unicode_form() == "NFC"
+
+
+def test_set_unicode_format():
+    set_unicode_form("NFD")
+    assert get_unicode_form() == "NFD"
+
+    set_unicode_form("NFC")
+    assert get_unicode_form() == "NFC"
+
+    # test invalid format
+    with pytest.raises(ValueError):
+        set_unicode_form("foo")
+
+    # Reset to correct format based
+    set_unicode_form(DEFAULT_UNICODE_FORM)
+
+
+def test_set_unicode_fs_format():
+    set_unicode_fs_form("NFC")
+    assert get_unicode_fs_form() == "NFC"
+
+    set_unicode_fs_form("NFD")
+    assert get_unicode_fs_form() == "NFD"
+
+    # test invalid format
+    with pytest.raises(ValueError):
+        set_unicode_fs_form("foo")
+
+    # Reset to correct format based on platform
+    set_unicode_fs_form("NFD" if is_macos else "NFC")
+
+
+def test_normalize_fs_path():
+    # Test with string path in NFC format
+    set_unicode_fs_form("NFC")
+    assert normalize_fs_path(UNICODE_PATH_NFD) == UNICODE_PATH_NFC
+
+    # Test with string path in NFD format
+    set_unicode_fs_form("NFD")
+    assert normalize_fs_path(UNICODE_PATH_NFC) == UNICODE_PATH_NFD
+
+    # Test with pathlib.Path object in NFC format
+    set_unicode_fs_form("NFC")
+    assert normalize_fs_path(pathlib.Path(UNICODE_PATH_NFD)) == pathlib.Path(
+        UNICODE_PATH_NFC
+    )
+
+    # Test with pathlib.Path object in NFD format
+    set_unicode_fs_form("NFD")
+    assert normalize_fs_path(pathlib.Path(UNICODE_PATH_NFC)) == pathlib.Path(
+        UNICODE_PATH_NFD
+    )
+
+    # Reset to correct format based on platform
+    set_unicode_fs_form("NFD" if is_macos else "NFC")
+
+
+def test_normalize_unicode():
+    # Test with str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(UNICODE_STR_NFD) == UNICODE_STR_NFC
+
+    # Test with str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(UNICODE_STR_NFC) == UNICODE_STR_NFD
+
+    # Test with list of str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(UNICODE_LIST_NFD) == UNICODE_LIST_NFC
+
+    # Test with list of str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(UNICODE_LIST_NFC) == UNICODE_LIST_NFD
+
+    # Test with tuple of str in NFC format
+    set_unicode_form("NFC")
+    assert normalize_unicode(tuple(UNICODE_LIST_NFD)) == tuple(UNICODE_LIST_NFC)
+
+    # Test with tuple of str in NFD format
+    set_unicode_form("NFD")
+    assert normalize_unicode(tuple(UNICODE_LIST_NFC)) == tuple(UNICODE_LIST_NFD)
+
+    # Test with None
+    assert normalize_unicode(None) is None
+
+    # Reset to correct format based
+    set_unicode_form(DEFAULT_UNICODE_FORM)

tests/test_uti.py

@@ -3,12 +3,12 @@
 import pytest
 
 import osxphotos.uti
+from osxphotos.platform import is_macos
 from osxphotos.uti import (
     _get_uti_from_mdls,
     get_preferred_uti_extension,
     get_uti_for_extension,
 )
-from osxphotos.utils import is_macos
 
 EXT_DICT = {"heic": "public.heic", "jpg": "public.jpeg", ".jpg": "public.jpeg"}
 UTI_DICT = {"public.heic": "heic", "public.jpeg": "jpeg"}