From 83d14ac191f9775e16515ca2fcafcfc1477a363d Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Mon, 16 Jan 2023 21:26:02 -0800 Subject: [PATCH] Feature add locations 929 (#932) * Initial implementation of add-locations command #929 * Updated help for add-locations * Fixed handling of shared albums in sync * Added test for add-locations --- osxphotos/cli/__init__.py | 2 + osxphotos/cli/add_locations.py | 190 ++++++++++++++++++++++++++++++++ osxphotos/cli/cli.py | 2 + tests/conftest.py | 101 ++++++++++++----- tests/test_cli_add_locations.py | 52 +++++++++ 5 files changed, 318 insertions(+), 29 deletions(-) create mode 100644 osxphotos/cli/add_locations.py create mode 100644 tests/test_cli_add_locations.py diff --git a/osxphotos/cli/__init__.py b/osxphotos/cli/__init__.py index 8e8af76a..d70a5075 100644 --- a/osxphotos/cli/__init__.py +++ b/osxphotos/cli/__init__.py @@ -44,6 +44,7 @@ if args.get("--debug", False): print("Debugging enabled", file=sys.stderr) from .about import about +from .add_locations import add_locations from .albums import albums from .cli import cli_main from .common import get_photos_db, load_uuid_from_file @@ -73,6 +74,7 @@ install_traceback() __all__ = [ "about", + "add_locations", "albums", "cli_main", "debug_dump", diff --git a/osxphotos/cli/add_locations.py b/osxphotos/cli/add_locations.py new file mode 100644 index 00000000..3fe42254 --- /dev/null +++ b/osxphotos/cli/add_locations.py @@ -0,0 +1,190 @@ +"""Add missing location data to photos in Photos.app using nearest neighbor.""" + +from __future__ import annotations + +import datetime + +import click +import photoscript + +import osxphotos +from osxphotos.utils import pluralize + +from .click_rich_echo import ( + rich_click_echo, + rich_echo_error, + set_rich_console, + set_rich_theme, + set_rich_timestamp, +) +from .color_themes import get_theme +from .common import QUERY_OPTIONS, THEME_OPTION, query_options_from_kwargs +from .param_types import TimeOffset +from .rich_progress import rich_progress +from .verbose import get_verbose_console, verbose_print + + +def get_location( + photos: list[osxphotos.PhotoInfo], idx: int, window: datetime.timedelta +) -> osxphotos.PhotoInfo | None: + """Find nearest neighbor with location data within window of time. + + Args: + photo: PhotoInfo object + idx: index of photo in list of photos + window: window of time to search for nearest neighbor + + Returns: + nearest neighbor PhotoInfo object or None if no neighbor found + """ + idx_back = None + idx_forward = None + if idx > 0: + # search backwards in time + for i in range(idx - 1, -1, -1): + if ( + photos[idx].date - photos[i].date <= window + and None not in photos[i].location + ): + idx_back = i + break + + if idx < len(photos) - 1: + # search forwards in time + for i in range(idx + 1, len(photos)): + if ( + photos[i].date - photos[idx].date <= window + and None not in photos[i].location + ): + idx_forward = i + break + + if idx_back is not None and idx_forward is not None: + # found location in both directions + # use location closest in time + if ( + photos[idx].date - photos[idx_back].date + < photos[idx_forward].date - photos[idx].date + ): + return photos[idx_back] + else: + return photos[idx_forward] + elif idx_back is not None: + return photos[idx_back] + elif idx_forward is not None: + return photos[idx_forward] + else: + return None + + +@click.command(name="add-locations") +@click.option( + "--window", + "-w", + type=TimeOffset(), + default="1 hr", + help="Window of time to search for nearest neighbor; " + "searches +/- window of time. Default is 1 hour. " + "Format is one of 'HH:MM:SS', 'D days', 'H hours' (or hr), 'M minutes' (or min), " + "'S seconds' (or sec), 'S' (where S is seconds).", +) +@click.option( + "--dry-run", + is_flag=True, + help="Don't actually add location, just print what would be done. " + "Most useful with --verbose.", +) +@click.option("--verbose", "-V", "verbose_", is_flag=True, help="Print verbose output.") +@click.option( + "--timestamp", "-T", is_flag=True, help="Add time stamp to verbose output." +) +@QUERY_OPTIONS +@THEME_OPTION +@click.pass_obj +@click.pass_context +def add_locations(ctx, cli_ob, window, dry_run, verbose_, timestamp, theme, **kwargs): + """Add missing location data to photos in Photos.app using nearest neighbor. + + This command will search for photos that are missing location data and look + for the nearest neighbor photo within a given window of time that contains + location information. If a photo is found within the window of time, the + location of the nearest neighbor will be used to update the location of the + photo. + + For example, if you took pictures with your iPhone and also with a camera that + doesn't have location information, you can use this command to add location + information to the photos taken with the camera from those taken with the + iPhone. + + If you have many photos with missing location information but no nearest neighbor + within the window of time, you could add location information to some photos manually + then run this command again to add location information to the remaining photos. + + You can specify a subset of photos to update using the query options. For example, + `--selected` to update only the selected photos, `--added-after 2020-01-01` to update + only photos added after Jan 1, 2020, etc. + + Example: + + Add location data to all photos with missing location data within a ±2 hour window: + + `osxphotos add-locations --window "2 hr" --verbose` + + The add-locations command assumes that photos already have the correct date and time. + If you have photos that are missing both location data and date/time information, + you can use `osxphotos timewarp` to add date/time information to the photos and then + use `osxphotos add-locations` to add location information. + See `osxphotos help timewarp` for more information. + """ + color_theme = get_theme(theme) + verbose = verbose_print( + verbose_, timestamp, rich=True, theme=color_theme, highlight=False + ) + # set console for rich_echo to be same as for verbose_ + set_rich_console(get_verbose_console()) + set_rich_theme(color_theme) + set_rich_timestamp(timestamp) + + verbose("Searching for photos with missing location data...") + + # load photos database + photosdb = osxphotos.PhotosDB(verbose=verbose) + query_options = query_options_from_kwargs(**kwargs) + photos = photosdb.query(query_options) + + # sort photos by date + photos = sorted(photos, key=lambda p: p.date) + + num_photos = len(photos) + missing_location = 0 + found_location = 0 + verbose(f"Processing {len(photos)} photos, window = ±{window}...") + with rich_progress(console=get_verbose_console(), mock=verbose_) as progress: + task = progress.add_task( + f"Processing [num]{num_photos}[/] {pluralize(len(photos), 'photo', 'photos')}, window = ±{window}", + total=num_photos, + ) + for idx, photo in enumerate(photos): + if None in photo.location: + missing_location += 1 + verbose( + f"Processing [filename]{photo.original_filename}[/] ([uuid]{photo.uuid}[/])" + ) + if neighbor := get_location(photos, idx, window): + verbose( + f"Adding location {neighbor.location} to [filename]{photo.original_filename}[/] ([uuid]{photo.uuid}[/])" + f" from [filename]{neighbor.original_filename}[/] ([uuid]{neighbor.uuid}[/])" + ) + found_location += 1 + if not dry_run: + photoscript.Photo(photo.uuid).location = neighbor.location + else: + verbose( + f"No location found for [filename]{photo.original_filename}[/] ([uuid]{photo.uuid}[/])" + ) + progress.advance(task) + rich_click_echo( + f"Done. Processed: [num]{num_photos}[/] photos, " + f"missing location: [num]{missing_location}[/], " + f"found location: [num]{found_location}[/] " + ) diff --git a/osxphotos/cli/cli.py b/osxphotos/cli/cli.py index 1e2e1384..c20b516e 100644 --- a/osxphotos/cli/cli.py +++ b/osxphotos/cli/cli.py @@ -11,6 +11,7 @@ from osxphotos._constants import PROFILE_SORT_KEYS from osxphotos._version import __version__ from .about import about +from .add_locations import add_locations from .albums import albums from .common import DB_OPTION, JSON_OPTION, OSXPHOTOS_HIDDEN from .debug_dump import debug_dump @@ -105,6 +106,7 @@ def cli_main(ctx, db, json_, profile, profile_sort, **kwargs): # install CLI commands for command in [ about, + add_locations, albums, debug_dump, diff, diff --git a/tests/conftest.py b/tests/conftest.py index 65e8dd24..01387f86 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,9 @@ """ pytest test configuration """ import os import pathlib +import shutil import tempfile +import time import photoscript import pytest @@ -21,6 +23,9 @@ TEST_IMPORT = False # run sync tests (configured with --test-sync) TEST_SYNC = False +# run add-locations tests (configured with --test-add-locations) +TEST_ADD_LOCATIONS = False + # don't clean up crash logs (configured with --no-cleanup) NO_CLEANUP = False @@ -51,11 +56,13 @@ if OS_VER == "15": TEST_LIBRARY_IMPORT = TEST_LIBRARY TEST_LIBRARY_SYNC = TEST_LIBRARY from tests.config_timewarp_catalina import TEST_LIBRARY_TIMEWARP + + TEST_LIBRARY_ADD_LOCATIONS = None else: TEST_LIBRARY = None TEST_LIBRARY_TIMEWARP = None TEST_LIBRARY_SYNC = None - # pytest.exit("This test suite currently only runs on MacOS Catalina ") + TEST_LIBRARY_ADD_LOCATIONS = "tests/Test-13.0.0.photoslibrary" @pytest.fixture(scope="session", autouse=True) @@ -79,6 +86,13 @@ def setup_photos_sync(): copy_photos_library(TEST_LIBRARY_SYNC, delay=10) +@pytest.fixture(scope="session", autouse=True) +def setup_photos_add_locations(): + if not TEST_ADD_LOCATIONS: + return + copy_photos_library(TEST_LIBRARY_ADD_LOCATIONS, delay=10) + + @pytest.fixture(autouse=True) def reset_singletons(): """Need to clean up any ExifTool singletons between tests""" @@ -107,6 +121,12 @@ def pytest_addoption(parser): default=False, help="run `osxphotos sync` tests", ) + parser.addoption( + "--test-add-locations", + action="store_true", + default=False, + help="run `osxphotos add-locations` tests", + ) parser.addoption( "--no-cleanup", action="store_true", @@ -144,6 +164,10 @@ def pytest_configure(config): config.addinivalue_line( "markers", "test_sync: mark test as requiring --test-sync to run" ) + config.addinivalue_line( + "markers", + "test_add_locations: mark test as requiring --test-add-locations to run", + ) # this is hacky but I can't figure out how to check config options in other fixtures if config.getoption("--timewarp"): @@ -158,6 +182,10 @@ def pytest_configure(config): global TEST_SYNC TEST_SYNC = True + if config.getoption("--test-add-locations"): + global TEST_ADD_LOCATIONS + TEST_ADD_LOCATIONS = True + if config.getoption("--no-cleanup"): global NO_CLEANUP NO_CLEANUP = True @@ -196,17 +224,25 @@ def pytest_collection_modifyitems(config, items): if "test_sync" in item.keywords: item.add_marker(skip_test_sync) + if not ( + config.getoption("--test-add-locations") + and TEST_LIBRARY_ADD_LOCATIONS is not None + ): + skip_test_sync = pytest.mark.skip( + reason="need --test-add-locations option and MacOS Ventura to run" + ) + for item in items: + if "test_add_locations" in item.keywords: + item.add_marker(skip_test_sync) -def copy_photos_library(photos_library, delay=0): + +def copy_photos_library(photos_library, delay=0, open=True): """copy the test library and open Photos, returns path to copied library""" - script = AppleScript( - """ - tell application "Photos" - quit - end tell - """ - ) - script.run() + + # quit Photos if it's running + photoslib = photoscript.PhotosLibrary() + photoslib.quit() + src = pathlib.Path(os.getcwd()) / photos_library picture_folder = ( pathlib.Path(os.environ["PHOTOSCRIPT_PICTURES_FOLDER"]) @@ -216,27 +252,34 @@ def copy_photos_library(photos_library, delay=0): picture_folder = picture_folder.expanduser() if not picture_folder.is_dir(): pytest.exit(f"Invalid picture folder: '{picture_folder}'") - dest = picture_folder / photos_library - ditto(src, dest) - script = AppleScript( - f""" - set tries to 0 - repeat while tries < 5 - try - tell application "Photos" - activate - delay 3 - open POSIX file "{dest}" - delay {delay} - end tell - set tries to 5 - on error - set tries to tries + 1 - end try - end repeat + dest = picture_folder / pathlib.Path(photos_library).name + + # copy src directory to dest directory, removing it if it already exists + shutil.rmtree(str(dest), ignore_errors=True) + + print(f"copying {src} to {picture_folder} ...") + copyFolder = AppleScript( + """ + on copyFolder(sourceFolder, destinationFolder) + -- sourceFolder and destinationFolder are strings of POSIX paths + set sourceFolder to POSIX file sourceFolder + set destinationFolder to POSIX file destinationFolder + tell application "Finder" + duplicate sourceFolder to destinationFolder + end tell + end copyFolder """ ) - script.run() + copyFolder.call("copyFolder", str(src), str(picture_folder)) + + # open Photos + if open: + # sometimes doesn't open the first time + time.sleep(delay) + photoslib.open(str(dest)) + time.sleep(delay) + photoslib.open(str(dest)) + return dest diff --git a/tests/test_cli_add_locations.py b/tests/test_cli_add_locations.py new file mode 100644 index 00000000..cb60bcdc --- /dev/null +++ b/tests/test_cli_add_locations.py @@ -0,0 +1,52 @@ +"""Test osxphotos add-locations command""" + +import photoscript +import pytest +from click.testing import CliRunner + +from osxphotos.cli.add_locations import add_locations + +UUID_TEST_PHOTO_1 = "F12384F6-CD17-4151-ACBA-AE0E3688539E" # Pumkins1.jpg +UUID_TEST_PHOTO_LOCATION = "D79B8D77-BFFC-460B-9312-034F2877D35B" # Pumkins2.jpg +TEST_LOCATION = (41.26067, -95.94056) # Omaha, NE + + +@pytest.mark.test_add_locations +def test_add_locations(): + """Test add-locations command""" + + with CliRunner().isolated_filesystem(): + # need to clear location data from test photo + test_photo = photoscript.Photo(UUID_TEST_PHOTO_1) + test_photo.location = None + source_photo = photoscript.Photo(UUID_TEST_PHOTO_LOCATION) + source_photo.location = TEST_LOCATION + + # now run add-locations + result = CliRunner().invoke( + add_locations, + [ + "--window", + "2 hours", + "--dry-run", + ], + ) + assert result.exit_code == 0 + # should find 3 matching locations: Pumkins1.jpg, Pumpkins3.jpg, Pumpkins4.jpg + assert "found location: 3" in result.output + + # verify location not really added + assert test_photo.location == (None, None) + + # run again without dry-run + result = CliRunner().invoke( + add_locations, + [ + "--window", + "2 hours", + ], + ) + assert result.exit_code == 0 + # should find 3 matching locations: Pumkins1.jpg, Pumpkins3.jpg, Pumpkins4.jpg + assert "found location: 3" in result.output + assert test_photo.location == TEST_LOCATION