From ebd878a075983ef3df0b1ead1a725e01508721f8 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Thu, 20 Aug 2020 06:39:48 -0700 Subject: [PATCH 1/6] Working on issue 206 --- osxphotos/fileutil.py | 43 ++++++++++++++++++--- osxphotos/photoinfo/_photoinfo_export.py | 49 ++++++++++++++++-------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/osxphotos/fileutil.py b/osxphotos/fileutil.py index 187d7ac8..9b6487d3 100644 --- a/osxphotos/fileutil.py +++ b/osxphotos/fileutil.py @@ -34,7 +34,12 @@ class FileUtilABC(ABC): @classmethod @abstractmethod - def cmp_sig(cls, file1, file2): + def cmp(cls, file1, file2, mtime1 = None): + pass + + @classmethod + @abstractmethod + def cmp_file_sig(cls, file1, file2): pass @classmethod @@ -114,11 +119,32 @@ class FileUtilMacOS(FileUtilABC): os.utime(path, times) @classmethod - def cmp_sig(cls, f1, s2): + def cmp(cls, f1, f2, mtime1 = None): + """Does shallow compare (file signatures) of f1 to file f2. + Arguments: + f1 -- File name + f2 -- File name + mtime1 -- optional, pass alternate file modification timestamp for f1; will be converted to int + + Return value: + True if the file signatures as returned by stat are the same, False otherwise. + Does not do a byte-by-byte comparison. + """ + + s1 = cls._sig(os.stat(f1)) + if mtime1 is not None: + s1 = (s1[0], s1[1], int(mtime1)) + s2 = cls._sig(os.stat(f2)) + if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG: + return False + return s1 == s2 + + @classmethod + def cmp_file_sig(cls, f1, s2): """Compare file f1 to signature s2. Arguments: f1 -- File name - s2 -- stats as returned by sig + s2 -- stats as returned by _sig Return value: True if the files are the same, False otherwise. @@ -140,7 +166,12 @@ class FileUtilMacOS(FileUtilABC): @staticmethod def _sig(st): - return (stat.S_IFMT(st.st_mode), st.st_size, st.st_mtime) + """ return tuple of (mode, size, mtime) of file based on os.stat + Args: + st: os.stat signature + """ + # use int(st.st_mtime) because ditto does not copy fractional portion of mtime + return (stat.S_IFMT(st.st_mode), st.st_size, int(st.st_mtime)) class FileUtil(FileUtilMacOS): @@ -151,8 +182,8 @@ class FileUtil(FileUtilMacOS): class FileUtilNoOp(FileUtil): """ No-Op implementation of FileUtil for testing / dry-run mode - all methods with exception of cmp_sig and file_cmp are no-op - cmp_sig functions as FileUtil.cmp_sig does + all methods with exception of cmp, cmp_file_sig and file_cmp are no-op + cmp and cmp_file_sig functions as FileUtil methods do file_cmp returns mock data """ diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index 2bb7660b..875c462f 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -11,7 +11,6 @@ # TODO: should this be its own PhotoExporter class? -import filecmp import glob import json import logging @@ -484,7 +483,7 @@ def export2( if update and dest.exists(): # destination exists, check to see if destination is the right UUID dest_uuid = export_db.get_uuid_for_file(dest) - if dest_uuid is None and filecmp.cmp(src, dest): + if dest_uuid is None and fileutil.cmp(src, dest): # might be exporting into a pre-ExportDB folder or the DB got deleted logging.debug( f"Found matching file with blank uuid: {self.uuid}, {dest}" @@ -516,7 +515,7 @@ def export2( dest = pathlib.Path(file_) found_match = True break - elif dest_uuid is None and filecmp.cmp(src, file_): + elif dest_uuid is None and fileutil.cmp(src, file_): # files match, update the UUID logging.debug( f"Found matching file with blank uuid: {self.uuid}, {file_}" @@ -836,21 +835,33 @@ def _export_photo( op_desc = "export_by_copying" if not update: - # not update, do the the hardlink - logging.debug(f"Not update: {op_desc} linking file {src} {dest}") + # not update, export the file + logging.debug(f"Exporting file with {op_desc} {src} {dest}") exported_files.append(dest_str) - else: #updating + else: # updating if not dest_exists: # update, destination doesn't exist (new file) logging.debug(f"Update: exporting new file with {op_desc} {src} {dest}") update_new_files.append(dest_str) else: # update, destination exists, but we might not need to replace it... - if (( export_as_hardlink and dest.samefile(src)) or - (not export_as_hardlink and not dest.samefile(src) and ( - ( exiftool and fileutil.cmp_sig(dest_str, export_db.get_stat_exif_for_file(dest_str))) or - (not exiftool and filecmp.cmp(src, dest))) - )): + if touch_file: + sig_cmp = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) + else: + sig_cmp = fileutil.cmp(src, dest) + if (export_as_hardlink and dest.samefile(src)) or ( + not export_as_hardlink + and not dest.samefile(src) + and ( + ( + exiftool + and fileutil.cmp_file_sig( + dest_str, export_db.get_stat_exif_for_file(dest_str) + ) + ) + or (not exiftool and sig_cmp) + ) + ): # destination exists but its signature is "identical" logging.debug(f"Update: skipping identical original files {src} {dest}") # call set_stat because code can reach this spot if no export DB but exporting a RAW or live photo @@ -859,13 +870,17 @@ def _export_photo( update_skipped_files.append(dest_str) else: # destination exists but is different - logging.debug(f"Update: removing existing file prior to {op_desc} {src} {dest}") + logging.debug( + f"Update: removing existing file prior to {op_desc} {src} {dest}" + ) update_updated_files.append(dest_str) if not update_skipped_files: if dest_exists and (update or overwrite): # need to remove the destination first - logging.debug(f"Update: removing existing file prior to export_as_hardlink {src} {dest}") + logging.debug( + f"Update: removing existing file prior to export_as_hardlink {src} {dest}" + ) # dest.unlink() fileutil.unlink(dest) if export_as_hardlink: @@ -873,7 +888,7 @@ def _export_photo( else: fileutil.copy(src, dest_str, norsrc=no_xattr) if touch_file: - ts=self.date.timestamp() + ts = self.date.timestamp() fileutil.utime(dest, (ts, ts)) export_db.set_data( @@ -886,7 +901,11 @@ def _export_photo( ) return ExportResults( - exported_files + update_new_files + update_updated_files, update_new_files, update_updated_files, update_skipped_files, [] + exported_files + update_new_files + update_updated_files, + update_new_files, + update_updated_files, + update_skipped_files, + [], ) From c9c920220545dc27c8cb1379d7bde15987cce72c Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Fri, 21 Aug 2020 05:53:52 -0700 Subject: [PATCH 2/6] Working on issue #206 --- osxphotos/__main__.py | 33 +++++++++++------ osxphotos/photoinfo/_photoinfo_export.py | 45 +++++++++++++++++------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index 8a609305..73714807 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -1543,6 +1543,7 @@ def export( results_updated = [] results_skipped = [] results_exif_updated = [] + results_touched = [] if verbose_: for p in photos: results = export_photo( @@ -1578,6 +1579,7 @@ def export( results_updated.extend(results.updated) results_skipped.extend(results.skipped) results_exif_updated.extend(results.exif_updated) + results_touched.extend(results.touched) else: # show progress bar @@ -1616,24 +1618,31 @@ def export( results_updated.extend(results.updated) results_skipped.extend(results.skipped) results_exif_updated.extend(results.exif_updated) + results_touched.extend(results.touched) stop_time = time.perf_counter() # print summary results if update: photo_str_new = "photos" if len(results_new) != 1 else "photo" photo_str_updated = "photos" if len(results_new) != 1 else "photo" photo_str_skipped = "photos" if len(results_skipped) != 1 else "photo" + photo_str_touched = "photos" if len(results_touched) != 1 else "photo" photo_str_exif_updated = ( "photos" if len(results_exif_updated) != 1 else "photo" ) - click.echo( - f"Exported: {len(results_new)} {photo_str_new}, " - + f"updated: {len(results_updated)} {photo_str_updated}, " - + f"skipped: {len(results_skipped)} {photo_str_skipped}, " - + f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}" - ) + summary = f"Exported: {len(results_new)} {photo_str_new}, " \ + f"updated: {len(results_updated)} {photo_str_updated}, " \ + f"skipped: {len(results_skipped)} {photo_str_skipped}, " \ + f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}" + if touch_file: + summary += f", touched date: {len(results_touched)} {photo_str_touched}" + click.echo(summary) else: photo_str = "photos" if len(results_exported) != 1 else "photo" - click.echo(f"Exported: {len(results_exported)} {photo_str}") + photo_str_touched = "photos" if len(results_touched) != 1 else "photo" + summary = f"Exported: {len(results_exported)} {photo_str}" + if touch_file: + summary += f", touched date: {len(results_touched)} {photo_str_touched}" + click.echo(summary) click.echo(f"Elapsed time: {(stop_time-start_time):.3f} seconds") else: click.echo("Did not find any photos to export") @@ -2125,25 +2134,26 @@ def export_photo( if photo.ismissing: space = " " if not verbose_ else "" verbose(f"{space}Skipping missing photo {photo.filename}") - return ExportResults([], [], [], [], []) + return ExportResults([], [], [], [], [], []) elif not os.path.exists(photo.path): space = " " if not verbose_ else "" verbose( f"{space}WARNING: file {photo.path} is missing but ismissing=False, " f"skipping {photo.filename}" ) - return ExportResults([], [], [], [], []) + return ExportResults([], [], [], [], [], []) elif photo.ismissing and not photo.iscloudasset or not photo.incloud: verbose( f"Skipping missing {photo.filename}: not iCloud asset or missing from cloud" ) - return ExportResults([], [], [], [], []) + return ExportResults([], [], [], [], [], []) results_exported = [] results_new = [] results_updated = [] results_skipped = [] results_exif_updated = [] + results_touched = [] filenames = get_filenames_from_template(photo, filename_template, original_name) for filename in filenames: @@ -2196,6 +2206,7 @@ def export_photo( results_updated.extend(export_results.updated) results_skipped.extend(export_results.skipped) results_exif_updated.extend(export_results.exif_updated) + results_touched.extend(export_results.touched) if verbose_: for exported in export_results.exported: @@ -2253,6 +2264,7 @@ def export_photo( results_updated.extend(export_results_edited.updated) results_skipped.extend(export_results_edited.skipped) results_exif_updated.extend(export_results_edited.exif_updated) + results_touched.extend(export_results.touched) if verbose_: for exported in export_results_edited.exported: @@ -2270,6 +2282,7 @@ def export_photo( results_updated, results_skipped, results_exif_updated, + results_touched ) diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index 875c462f..33bd3a47 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -36,7 +36,8 @@ from ..fileutil import FileUtil from ..utils import dd_to_dms_str, findfiles ExportResults = namedtuple( - "ExportResults", ["exported", "new", "updated", "skipped", "exif_updated"] + "ExportResults", + ["exported", "new", "updated", "skipped", "exif_updated", "touched"], ) @@ -376,6 +377,9 @@ def export2( # list of all files skipped because they do not need to be updated (for use with update=True) update_skipped_files = [] + # list of all files with utime touched (touch_file = True) + touched_files = [] + # check edited and raise exception trying to export edited version of # photo that hasn't been edited if edited and not self.hasadjustments: @@ -566,6 +570,7 @@ def export2( update_new_files = results.new update_updated_files = results.updated update_skipped_files = results.skipped + touched_files = results.touched # copy live photo associated .mov if requested if live_photo and self.live_photo: @@ -592,6 +597,7 @@ def export2( update_new_files.extend(results.new) update_updated_files.extend(results.updated) update_skipped_files.extend(results.skipped) + touched_files.extend(results.touched) else: logging.debug(f"Skipping missing live movie for {filename}") @@ -618,6 +624,7 @@ def export2( update_new_files.extend(results.new) update_updated_files.extend(results.updated) update_skipped_files.extend(results.skipped) + touched_files.extend(results.touched) else: logging.debug(f"Skipping missing RAW photo for {filename}") else: @@ -662,6 +669,7 @@ def export2( ) if exported is not None: + # TODO: add touch_file code exported_files.extend(exported) else: logging.warning( @@ -704,6 +712,7 @@ def export2( raise e # if exiftool, write the metadata + # TODO: add touch_file code to update touch time if update: exif_files = update_new_files + update_updated_files + update_skipped_files else: @@ -784,6 +793,7 @@ def export2( update_updated_files, update_skipped_files, exif_files_updated, + touched_files, ) @@ -826,6 +836,7 @@ def _export_photo( update_updated_files = [] update_new_files = [] update_skipped_files = [] + touched_files = [] dest_str = str(dest) dest_exists = dest.exists() @@ -838,11 +849,15 @@ def _export_photo( # not update, export the file logging.debug(f"Exporting file with {op_desc} {src} {dest}") exported_files.append(dest_str) + if touch_file: + touched_files.append(dest_str) else: # updating if not dest_exists: # update, destination doesn't exist (new file) logging.debug(f"Update: exporting new file with {op_desc} {src} {dest}") update_new_files.append(dest_str) + if touch_file: + touched_files.append(dest_str) else: # update, destination exists, but we might not need to replace it... if touch_file: @@ -863,33 +878,38 @@ def _export_photo( ) ): # destination exists but its signature is "identical" - logging.debug(f"Update: skipping identical original files {src} {dest}") # call set_stat because code can reach this spot if no export DB but exporting a RAW or live photo # potentially re-writes the data in the database but ensures database is complete export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str)) update_skipped_files.append(dest_str) else: - # destination exists but is different - logging.debug( - f"Update: removing existing file prior to {op_desc} {src} {dest}" - ) - update_updated_files.append(dest_str) + # destination exists but signature is different + if touch_file and fileutil.cmp(src, dest) and not sig_cmp: + # destination exists, signature matches original but does not match expected touch time + # skip exporting but update touch time + update_skipped_files.append(dest_str) + touched_files.append(dest_str) + else: + # destination exists but is different + update_updated_files.append(dest_str) + if touch_file: + touched_files.append(dest_str) if not update_skipped_files: if dest_exists and (update or overwrite): # need to remove the destination first logging.debug( - f"Update: removing existing file prior to export_as_hardlink {src} {dest}" + f"Update: removing existing file prior to {op_desc} {src} {dest}" ) - # dest.unlink() fileutil.unlink(dest) if export_as_hardlink: fileutil.hardlink(src, dest) else: fileutil.copy(src, dest_str, norsrc=no_xattr) - if touch_file: - ts = self.date.timestamp() - fileutil.utime(dest, (ts, ts)) + + if touched_files: + ts = self.date.timestamp() + fileutil.utime(dest, (ts, ts)) export_db.set_data( dest_str, @@ -906,6 +926,7 @@ def _export_photo( update_updated_files, update_skipped_files, [], + touched_files, ) From 6c11e3fa5b5b05b98b9fdbb0e59e3a78c7dff980 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sat, 22 Aug 2020 08:12:26 -0700 Subject: [PATCH 3/6] --touch-file now working with --update --- osxphotos/__main__.py | 4 +- osxphotos/photoinfo/_photoinfo_export.py | 23 ++-- tests/test_cli.py | 166 ++++++++++++++++++++++- 3 files changed, 179 insertions(+), 14 deletions(-) diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index 73714807..fc141185 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -1623,7 +1623,7 @@ def export( # print summary results if update: photo_str_new = "photos" if len(results_new) != 1 else "photo" - photo_str_updated = "photos" if len(results_new) != 1 else "photo" + photo_str_updated = "photos" if len(results_updated) != 1 else "photo" photo_str_skipped = "photos" if len(results_skipped) != 1 else "photo" photo_str_touched = "photos" if len(results_touched) != 1 else "photo" photo_str_exif_updated = ( @@ -2217,6 +2217,8 @@ def export_photo( verbose(f"Exported updated file {updated}") for skipped in export_results.skipped: verbose(f"Skipped up to date file {skipped}") + for touched in export_results.touched: + verbose(f"Touched date on file {touched}") # if export-edited, also export the edited version # verify the photo has adjustments and valid path to avoid raising an exception diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index 33bd3a47..49d1837e 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -850,7 +850,10 @@ def _export_photo( logging.debug(f"Exporting file with {op_desc} {src} {dest}") exported_files.append(dest_str) if touch_file: - touched_files.append(dest_str) + sig = fileutil.file_sig(src) + sig = (sig[0], sig[1], self.date.timestamp()) + if not fileutil.cmp_file_sig(src, sig): + touched_files.append(dest_str) else: # updating if not dest_exists: # update, destination doesn't exist (new file) @@ -860,10 +863,9 @@ def _export_photo( touched_files.append(dest_str) else: # update, destination exists, but we might not need to replace it... - if touch_file: - sig_cmp = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) - else: - sig_cmp = fileutil.cmp(src, dest) + cmp_orig = fileutil.cmp(src, dest) + cmp_touch = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) + sig_cmp = cmp_touch if touch_file else cmp_orig if (export_as_hardlink and dest.samefile(src)) or ( not export_as_hardlink and not dest.samefile(src) @@ -877,18 +879,19 @@ def _export_photo( or (not exiftool and sig_cmp) ) ): - # destination exists but its signature is "identical" - # call set_stat because code can reach this spot if no export DB but exporting a RAW or live photo - # potentially re-writes the data in the database but ensures database is complete - export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str)) update_skipped_files.append(dest_str) else: # destination exists but signature is different - if touch_file and fileutil.cmp(src, dest) and not sig_cmp: + if touch_file and cmp_orig and not cmp_touch: # destination exists, signature matches original but does not match expected touch time # skip exporting but update touch time update_skipped_files.append(dest_str) touched_files.append(dest_str) + elif not touch_file and cmp_touch and not cmp_orig: + # destination exists, signature matches expected touch but not original + # user likely exported with touch_file and is now exporting without touch_file + # don't update the file because it's same but leave touch time + update_skipped_files.append(dest_str) else: # destination exists but is different update_updated_files.append(dest_str) diff --git a/tests/test_cli.py b/tests/test_cli.py index 83ddffba..8143f36c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,6 +14,7 @@ PLACES_PHOTOS_DB = "tests/Test-Places-Catalina-10_15_1.photoslibrary" PLACES_PHOTOS_DB_13 = "tests/Test-Places-High-Sierra-10.13.6.photoslibrary" PHOTOS_DB_15_4 = "tests/Test-10.15.4.photoslibrary" PHOTOS_DB_15_5 = "tests/Test-10.15.5.photoslibrary" +PHOTOS_DB_15_6 = "tests/Test-10.15.5.photoslibrary" PHOTOS_DB_14_6 = "tests/Test-10.14.6.photoslibrary" UUID_FILE = "tests/uuid_from_file.txt" @@ -182,6 +183,7 @@ CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B" CLI_EXPORT_UUID_FILENAME = "Pumkins2.jpg" +CLI_EXPORT_BY_DATE_TOUCH_TIMES = [1538165373, 1538163349] CLI_EXPORT_BY_DATE = ["2018/09/28/Pumpkins3.jpg", "2018/09/28/Pumkins1.jpg"] CLI_EXPORT_SIDECAR_FILENAMES = ["Pumkins2.jpg", "Pumkins2.json", "Pumkins2.xmp"] @@ -706,7 +708,7 @@ def test_query_date_1(): import time from osxphotos.__main__ import query - os.environ['TZ'] = "US/Pacific" + os.environ["TZ"] = "US/Pacific" time.tzset() runner = CliRunner() @@ -726,6 +728,7 @@ def test_query_date_1(): json_got = json.loads(result.output) assert len(json_got) == 4 + def test_query_date_2(): """ Test --from-date and --to-date """ import json @@ -735,7 +738,7 @@ def test_query_date_2(): import time from osxphotos.__main__ import query - os.environ['TZ'] = "Asia/Jerusalem" + os.environ["TZ"] = "Asia/Jerusalem" time.tzset() runner = CliRunner() @@ -755,6 +758,7 @@ def test_query_date_2(): json_got = json.loads(result.output) assert len(json_got) == 2 + def test_query_date_timezone(): """ Test --from-date, --to-date with ISO 8601 timezone """ import json @@ -764,7 +768,7 @@ def test_query_date_timezone(): import time from osxphotos.__main__ import query - os.environ['TZ'] = "US/Pacific" + os.environ["TZ"] = "US/Pacific" time.tzset() runner = CliRunner() @@ -2587,6 +2591,162 @@ def test_export_directory_template_1_dry_run(): assert not os.path.isfile(os.path.join(workdir, filepath)) +def test_export_touch_files(): + """ test export with --touch-files """ + import os + + import osxphotos + from osxphotos.__main__ import export + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "-V", + "--touch-file", + "--export-by-date", + ], + ) + assert result.exit_code == 0 + + assert "Exported: 16 photos, touched date: 14 photos" in result.output + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) == mtime + + +def test_export_touch_files_update(): + """ test complex export scenario with --update and --touch-files """ + import os + import pathlib + import time + + import osxphotos + from osxphotos.__main__ import export + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + # basic export with dry-run + result = runner.invoke( + export, + [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--dry-run"], + ) + assert result.exit_code == 0 + + assert "Exported: 16 photos" in result.output + + assert not pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file() + + # without dry-run + result = runner.invoke( + export, [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date"] + ) + assert result.exit_code == 0 + + assert "Exported: 16 photos" in result.output + + assert pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file() + + # --update + result = runner.invoke( + export, + [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos" + in result.output + ) + + # --update --touch-file --dry-run + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--touch-file", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 14 photos" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) != mtime + + # --update --touch-file + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--touch-file", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 14 photos" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) == mtime + + # touch one file and run update again + ts = time.time() + os.utime(CLI_EXPORT_BY_DATE[0], (ts, ts)) + + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--touch-file", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 1 photo, skipped: 15 photos, updated EXIF data: 0 photos, touched date: 1 photo" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) == mtime + + # run update without --touch-file + result = runner.invoke( + export, + [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos" + in result.output + ) + + def test_labels(): """Test osxphotos labels """ import json From 9f6426275739acc919a897a564d0a3611883aa85 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sun, 23 Aug 2020 08:27:21 -0700 Subject: [PATCH 4/6] Finished --touch-file, closes #206 --- README.md | 18 +- osxphotos/__main__.py | 41 +++-- osxphotos/_export_db.py | 14 +- osxphotos/_version.py | 2 +- osxphotos/fileutil.py | 4 +- osxphotos/photoinfo/_photoinfo_export.py | 75 +++++---- tests/test_cli.py | 201 +++++++++++++++++++++++ 7 files changed, 300 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index bd87608e..c74c84ef 100644 --- a/README.md +++ b/README.md @@ -203,14 +203,14 @@ Options: both images and movies). --only-photos Search only for photos/images (default searches both images and movies). - --from-date [%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S] - Search by start item date, e.g. - 2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 - w/o TZ). - --to-date [%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S] - Search by end item date, e.g. - 2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 - w/o TZ). + --from-date DATETIME Search by start item date, e.g. + 2000-01-12T12:00:00, + 2001-01-12T12:00:00-07:00, or 2000-12-31 + (ISO 8601). + --to-date DATETIME Search by end item date, e.g. + 2000-01-12T12:00:00, + 2001-01-12T12:00:00-07:00, or 2000-12-31 + (ISO 8601). --deleted Include photos from the 'Recently Deleted' folder. --deleted-only Include only photos from the 'Recently @@ -222,6 +222,8 @@ Options: --export-as-hardlink Hardlink files instead of copying them. Cannot be used with --exiftool which creates copies of the files with embedded EXIF data. + --touch-file Sets the file's modification time to match + photo date. --overwrite Overwrite existing files. Default behavior is to add (1), (2), etc to filename if file already exists. Use this with caution as it diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index fc141185..8e4749f4 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -1124,7 +1124,7 @@ def query( @click.option( "--touch-file", is_flag=True, - help="Sets the file's modification time upon photo date.", + help="Sets the file's modification time to match photo date.", ) @click.option( "--overwrite", @@ -1269,6 +1269,13 @@ def query( "to a filesystem that doesn't support Mac OS extended attributes. Only use this if you get " "an error while exporting.", ) +@click.option( + "--use-photos-export", + is_flag=True, + default=False, + hidden=True, + help="Force the use of AppleScript to export even if not missing (see also --download-missing).", +) @DB_ARGUMENT @click.argument("dest", nargs=1, type=click.Path(exists=True)) @click.pass_obj @@ -1350,6 +1357,7 @@ def export( label, deleted, deleted_only, + use_photos_export, ): """ Export photos from the Photos database. Export path DEST is required. @@ -1520,7 +1528,6 @@ def export( deleted_only=deleted_only, ) - results_exported = [] if photos: if export_bursts: # add the burst_photos to the export set @@ -1539,6 +1546,7 @@ def export( # because the original code used --original-name as an option original_name = not current_name + results_exported = [] results_new = [] results_updated = [] results_skipped = [] @@ -1573,6 +1581,7 @@ def export( dry_run=dry_run, touch_file=touch_file, edited_suffix=edited_suffix, + use_photos_export=use_photos_export, ) results_exported.extend(results.exported) results_new.extend(results.new) @@ -1612,6 +1621,7 @@ def export( dry_run=dry_run, touch_file=touch_file, edited_suffix=edited_suffix, + use_photos_export=use_photos_export, ) results_exported.extend(results.exported) results_new.extend(results.new) @@ -1629,10 +1639,12 @@ def export( photo_str_exif_updated = ( "photos" if len(results_exif_updated) != 1 else "photo" ) - summary = f"Exported: {len(results_new)} {photo_str_new}, " \ - f"updated: {len(results_updated)} {photo_str_updated}, " \ - f"skipped: {len(results_skipped)} {photo_str_skipped}, " \ - f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}" + summary = ( + f"Exported: {len(results_new)} {photo_str_new}, " + f"updated: {len(results_updated)} {photo_str_updated}, " + f"skipped: {len(results_skipped)} {photo_str_skipped}, " + f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}" + ) if touch_file: summary += f", touched date: {len(results_touched)} {photo_str_touched}" click.echo(summary) @@ -2092,6 +2104,7 @@ def export_photo( dry_run=None, touch_file=None, edited_suffix="_edited", + use_photos_export=False, ): """ Helper function for export that does the actual export @@ -2119,7 +2132,8 @@ def export_photo( export_db: export database instance compatible with ExportDB_ABC fileutil: file util class compatible with FileUtilABC dry_run: boolean; if True, doesn't actually export or update any files - touch_file: boolean; sets file's modification time upon photo date + touch_file: boolean; sets file's modification time to match photo date + use_photos_export: boolean; if True forces the use of AppleScript to export even if photo not missing Returns: list of path(s) of exported photo or None if photo was missing @@ -2172,8 +2186,10 @@ def export_photo( # if download_missing and the photo is missing or path doesn't exist, # try to download with Photos - use_photos_export = download_missing and ( - photo.ismissing or not os.path.exists(photo.path) + use_photos_export = ( + download_missing and (photo.ismissing or not os.path.exists(photo.path)) + if not use_photos_export + else True ) # export the photo to each path in dest_paths @@ -2225,7 +2241,6 @@ def export_photo( if export_edited and photo.hasadjustments: # if download_missing and the photo is missing or path doesn't exist, # try to download with Photos - use_photos_export = download_missing and photo.path_edited is None if not download_missing and photo.path_edited is None: verbose(f"Skipping missing edited photo for {filename}") else: @@ -2266,7 +2281,7 @@ def export_photo( results_updated.extend(export_results_edited.updated) results_skipped.extend(export_results_edited.skipped) results_exif_updated.extend(export_results_edited.exif_updated) - results_touched.extend(export_results.touched) + results_touched.extend(export_results_edited.touched) if verbose_: for exported in export_results_edited.exported: @@ -2277,6 +2292,8 @@ def export_photo( verbose(f"Exported updated file {updated}") for skipped in export_results_edited.skipped: verbose(f"Skipped up to date file {skipped}") + for touched in export_results_edited.touched: + verbose(f"Touched date on file {touched}") return ExportResults( results_exported, @@ -2284,7 +2301,7 @@ def export_photo( results_updated, results_skipped, results_exif_updated, - results_touched + results_touched, ) diff --git a/osxphotos/_export_db.py b/osxphotos/_export_db.py index 1eaf944d..23726218 100644 --- a/osxphotos/_export_db.py +++ b/osxphotos/_export_db.py @@ -189,7 +189,12 @@ class ExportDB(ExportDB_ABC): (filename,), ) results = c.fetchone() - stats = results[0:3] if results else None + if results: + stats = results[0:3] + mtime = int(stats[2]) if stats[2] is not None else None + stats = (stats[0], stats[1], mtime) + else: + stats = (None, None, None) except Error as e: logging.warning(e) stats = (None, None, None) @@ -232,7 +237,12 @@ class ExportDB(ExportDB_ABC): (filename,), ) results = c.fetchone() - stats = results[0:3] if results else None + if results: + stats = results[0:3] + mtime = int(stats[2]) if stats[2] is not None else None + stats = (stats[0], stats[1], mtime) + else: + stats = (None, None, None) except Error as e: logging.warning(e) stats = (None, None, None) diff --git a/osxphotos/_version.py b/osxphotos/_version.py index 19d85187..9f9a3090 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,3 +1,3 @@ """ version info """ -__version__ = "0.33.1" +__version__ = "0.33.2" diff --git a/osxphotos/fileutil.py b/osxphotos/fileutil.py index 9b6487d3..f5609e09 100644 --- a/osxphotos/fileutil.py +++ b/osxphotos/fileutil.py @@ -34,7 +34,7 @@ class FileUtilABC(ABC): @classmethod @abstractmethod - def cmp(cls, file1, file2, mtime1 = None): + def cmp(cls, file1, file2, mtime1=None): pass @classmethod @@ -119,7 +119,7 @@ class FileUtilMacOS(FileUtilABC): os.utime(path, times) @classmethod - def cmp(cls, f1, f2, mtime1 = None): + def cmp(cls, f1, f2, mtime1=None): """Does shallow compare (file signatures) of f1 to file f2. Arguments: f1 -- File name diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index 49d1837e..cf52dfcb 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -629,7 +629,7 @@ def export2( logging.debug(f"Skipping missing RAW photo for {filename}") else: # use_photo_export - exported = None + exported = [] # export live_photo .mov file? live_photo = True if live_photo and self.live_photo else False if edited: @@ -639,7 +639,7 @@ def export2( filestem = dest.stem else: # didn't get passed a filename, add _edited - filestem = f"{dest.stem}_edited" + filestem = f"{dest.stem}{edited_identifier}" dest = dest.parent / f"{filestem}.jpeg" exported = _export_photo_uuid_applescript( @@ -668,9 +668,16 @@ def export2( dry_run=dry_run, ) - if exported is not None: - # TODO: add touch_file code + if exported: + if touch_file: + for exported_file in exported: + touched_files.append(exported_file) + ts = int(self.date.timestamp()) + fileutil.utime(exported_file, (ts, ts)) exported_files.extend(exported) + if update: + update_new_files.extend(exported) + else: logging.warning( f"Error exporting photo {self.uuid} to {dest} with use_photos_export" @@ -712,7 +719,6 @@ def export2( raise e # if exiftool, write the metadata - # TODO: add touch_file code to update touch time if update: exif_files = update_new_files + update_updated_files + update_skipped_files else: @@ -773,6 +779,7 @@ def export2( keyword_template=keyword_template, description_template=description_template, ) + export_db.set_exifdata_for_file( exported_file, self._exiftool_json_sidecar( @@ -787,7 +794,15 @@ def export2( ) exif_files_updated.append(exported_file) - return ExportResults( + if touch_file: + for exif_file in exif_files_updated: + touched_files.append(exported_file) + ts = int(self.date.timestamp()) + fileutil.utime(exported_file, (ts, ts)) + + touched_files = list(set(touched_files)) + + results = ExportResults( exported_files, update_new_files, update_updated_files, @@ -795,6 +810,7 @@ def export2( exif_files_updated, touched_files, ) + return results def _export_photo( @@ -863,22 +879,21 @@ def _export_photo( touched_files.append(dest_str) else: # update, destination exists, but we might not need to replace it... - cmp_orig = fileutil.cmp(src, dest) - cmp_touch = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) + if exiftool: + sig_exif = export_db.get_stat_exif_for_file(dest_str) + cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif) + sig_exif = (sig_exif[0], sig_exif[1], int(self.date.timestamp())) + cmp_touch = fileutil.cmp_file_sig(dest_str, sig_exif) + else: + cmp_orig = fileutil.cmp(src, dest) + cmp_touch = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) + sig_cmp = cmp_touch if touch_file else cmp_orig + if (export_as_hardlink and dest.samefile(src)) or ( - not export_as_hardlink - and not dest.samefile(src) - and ( - ( - exiftool - and fileutil.cmp_file_sig( - dest_str, export_db.get_stat_exif_for_file(dest_str) - ) - ) - or (not exiftool and sig_cmp) - ) + not export_as_hardlink and not dest.samefile(src) and sig_cmp ): + # destination exists and signatures match, skip it update_skipped_files.append(dest_str) else: # destination exists but signature is different @@ -910,18 +925,18 @@ def _export_photo( else: fileutil.copy(src, dest_str, norsrc=no_xattr) - if touched_files: - ts = self.date.timestamp() - fileutil.utime(dest, (ts, ts)) + export_db.set_data( + dest_str, + self.uuid, + fileutil.file_sig(dest_str), + (None, None, None), + self.json(), + None, + ) - export_db.set_data( - dest_str, - self.uuid, - fileutil.file_sig(dest_str), - (None, None, None), - self.json(), - None, - ) + if touched_files: + ts = int(self.date.timestamp()) + fileutil.utime(dest, (ts, ts)) return ExportResults( exported_files + update_new_files + update_updated_files, diff --git a/tests/test_cli.py b/tests/test_cli.py index 8143f36c..d745efd3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2304,6 +2304,16 @@ def test_export_update_exiftool(): in result.output ) + # update with exiftool again, should be no changes + result = runner.invoke( + export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "--update", "--exiftool"] + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 8 photos, updated EXIF data: 0 photos" + in result.output + ) + def test_export_update_hardlink(): """ test export with hardlink then update """ @@ -2747,6 +2757,197 @@ def test_export_touch_files_update(): ) +@pytest.mark.skipif(exiftool is None, reason="exiftool not installed") +def test_export_touch_files_exiftool_update(): + """ test complex export scenario with --update, --exiftol, and --touch-files """ + import os + import pathlib + import time + + import osxphotos + from osxphotos.__main__ import export + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + # basic export with dry-run + result = runner.invoke( + export, + [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--dry-run"], + ) + assert result.exit_code == 0 + + assert "Exported: 16 photos" in result.output + + assert not pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file() + + # without dry-run + result = runner.invoke( + export, [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date"] + ) + assert result.exit_code == 0 + + assert "Exported: 16 photos" in result.output + + assert pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file() + + # --update + result = runner.invoke( + export, + [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos" + in result.output + ) + + # --update --exiftool --dry-run + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + "--dry-run", + ], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 16 photos, skipped: 0 photos, updated EXIF data: 16 photos" + in result.output + ) + + # --update --exiftool + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + ], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 16 photos, skipped: 0 photos, updated EXIF data: 16 photos" + in result.output + ) + + # --update --touch-file --exiftool --dry-run + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + "--touch-file", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 16 photos" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) != mtime + + # --update --touch-file --exiftool + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + "--touch-file", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 16 photos" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) == mtime + + # touch one file and run update again + ts = time.time() + os.utime(CLI_EXPORT_BY_DATE[0], (ts, ts)) + + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + "--touch-file", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 1 photo, skipped: 15 photos, updated EXIF data: 1 photo, touched date: 1 photo" + in result.output + ) + + for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + st = os.stat(fname) + assert int(st.st_mtime) == mtime + + # run --update --exiftool --touch-file again + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--update", + "--exiftool", + "--touch-file", + ], + ) + assert result.exit_code == 0 + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 0 photos" + in result.output + ) + + # run update without --touch-file + result = runner.invoke( + export, + [ + os.path.join(cwd, PHOTOS_DB_15_6), + ".", + "--export-by-date", + "--exiftool", + "--update", + ], + ) + assert result.exit_code == 0 + + assert ( + "Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos" + in result.output + ) + + def test_labels(): """Test osxphotos labels """ import json From 6e5ea8e013860d9bfeb83b2b7d051591e8fb76d5 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sun, 23 Aug 2020 08:37:12 -0700 Subject: [PATCH 5/6] Fixed touch tests to use correct timezone --- tests/test_cli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index d745efd3..5e051b34 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2604,10 +2604,14 @@ def test_export_directory_template_1_dry_run(): def test_export_touch_files(): """ test export with --touch-files """ import os + import time import osxphotos from osxphotos.__main__ import export + os.environ["TZ"] = "US/Pacific" + time.tzset() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager @@ -2640,6 +2644,9 @@ def test_export_touch_files_update(): import osxphotos from osxphotos.__main__ import export + os.environ["TZ"] = "US/Pacific" + time.tzset() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager @@ -2767,6 +2774,9 @@ def test_export_touch_files_exiftool_update(): import osxphotos from osxphotos.__main__ import export + os.environ["TZ"] = "US/Pacific" + time.tzset() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager From 1bf7105737fbd756064a2f9ef4d4bbd0b067978c Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sun, 23 Aug 2020 11:06:01 -0700 Subject: [PATCH 6/6] Fixed touch tests --- osxphotos/photoinfo/_photoinfo_export.py | 4 +- tests/test_cli.py | 127 +++++++++++++++++------ 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/osxphotos/photoinfo/_photoinfo_export.py b/osxphotos/photoinfo/_photoinfo_export.py index cf52dfcb..71b0e353 100644 --- a/osxphotos/photoinfo/_photoinfo_export.py +++ b/osxphotos/photoinfo/_photoinfo_export.py @@ -867,7 +867,7 @@ def _export_photo( exported_files.append(dest_str) if touch_file: sig = fileutil.file_sig(src) - sig = (sig[0], sig[1], self.date.timestamp()) + sig = (sig[0], sig[1], int(self.date.timestamp())) if not fileutil.cmp_file_sig(src, sig): touched_files.append(dest_str) else: # updating @@ -886,7 +886,7 @@ def _export_photo( cmp_touch = fileutil.cmp_file_sig(dest_str, sig_exif) else: cmp_orig = fileutil.cmp(src, dest) - cmp_touch = fileutil.cmp(src, dest, mtime1=self.date.timestamp()) + cmp_touch = fileutil.cmp(src, dest, mtime1=int(self.date.timestamp())) sig_cmp = cmp_touch if touch_file else cmp_orig diff --git a/tests/test_cli.py b/tests/test_cli.py index 5e051b34..065f932d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,7 +14,8 @@ PLACES_PHOTOS_DB = "tests/Test-Places-Catalina-10_15_1.photoslibrary" PLACES_PHOTOS_DB_13 = "tests/Test-Places-High-Sierra-10.13.6.photoslibrary" PHOTOS_DB_15_4 = "tests/Test-10.15.4.photoslibrary" PHOTOS_DB_15_5 = "tests/Test-10.15.5.photoslibrary" -PHOTOS_DB_15_6 = "tests/Test-10.15.5.photoslibrary" +PHOTOS_DB_15_6 = "tests/Test-10.15.6.photoslibrary" +PHOTOS_DB_TOUCH = PHOTOS_DB_15_6 PHOTOS_DB_14_6 = "tests/Test-10.14.6.photoslibrary" UUID_FILE = "tests/uuid_from_file.txt" @@ -183,7 +184,20 @@ CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B" CLI_EXPORT_UUID_FILENAME = "Pumkins2.jpg" +CLI_EXPORT_BY_DATE_TOUCH_UUID = [ + "1EB2B765-0765-43BA-A90C-0D0580E6172C", + "F12384F6-CD17-4151-ACBA-AE0E3688539E", +] CLI_EXPORT_BY_DATE_TOUCH_TIMES = [1538165373, 1538163349] +CLI_EXPORT_BY_DATE_NEED_TOUCH = [ + "2018/09/28/Pumkins2.jpg", + "2018/10/13/St James Park.jpg", +] +CLI_EXPORT_BY_DATE_NEED_TOUCH_UUID = [ + "D79B8D77-BFFC-460B-9312-034F2877D35B", + "DC99FBDD-7A52-4100-A5BB-344131646C30", +] +CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES = [1538165227, 1539436692] CLI_EXPORT_BY_DATE = ["2018/09/28/Pumpkins3.jpg", "2018/09/28/Pumkins1.jpg"] CLI_EXPORT_SIDECAR_FILENAMES = ["Pumkins2.jpg", "Pumkins2.json", "Pumkins2.xmp"] @@ -325,6 +339,55 @@ except: exiftool = None +def touch_all_photos_in_db(dbpath): + """ touch date on all photos in a library + helper function for --touch-file tests + + Args: + dbpath: path to photos library to touch + """ + import os + import time + + import osxphotos + + ts = int(time.time()) + for photo in osxphotos.PhotosDB(dbpath).photos(): + if photo.path is not None: + os.utime(photo.path, (ts, ts)) + if photo.path_edited is not None: + os.utime(photo.path_edited, (ts, ts)) + if photo.path_raw is not None: + os.utime(photo.path_raw, (ts, ts)) + if photo.path_live_photo is not None: + os.utime(photo.path_live_photo, (ts, ts)) + + +def setup_touch_tests(): + """ perform setup needed for --touch-file tests """ + import os + import time + import logging + import osxphotos + + touch_all_photos_in_db(PHOTOS_DB_TOUCH) + + photos = osxphotos.PhotosDB(PHOTOS_DB_TOUCH).photos_by_uuid( + CLI_EXPORT_BY_DATE_TOUCH_UUID + ) + for photo in photos: + logging.warning(photo.path) + ts = int(photo.date.timestamp()) + if photo.path is not None: + os.utime(photo.path, (ts, ts)) + if photo.path_edited is not None: + os.utime(photo.path_edited, (ts, ts)) + if photo.path_raw is not None: + os.utime(photo.path_raw, (ts, ts)) + if photo.path_live_photo is not None: + os.utime(photo.path_live_photo, (ts, ts)) + + def test_osxphotos(): import osxphotos from osxphotos.__main__ import cli @@ -2612,6 +2675,8 @@ def test_export_touch_files(): os.environ["TZ"] = "US/Pacific" time.tzset() + setup_touch_tests() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager @@ -2619,7 +2684,7 @@ def test_export_touch_files(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "-V", "--touch-file", @@ -2632,7 +2697,7 @@ def test_export_touch_files(): for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) == mtime + assert int(st.st_mtime) == int(mtime) def test_export_touch_files_update(): @@ -2647,6 +2712,8 @@ def test_export_touch_files_update(): os.environ["TZ"] = "US/Pacific" time.tzset() + setup_touch_tests() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager @@ -2654,7 +2721,7 @@ def test_export_touch_files_update(): # basic export with dry-run result = runner.invoke( export, - [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--dry-run"], + [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--dry-run"], ) assert result.exit_code == 0 @@ -2664,7 +2731,7 @@ def test_export_touch_files_update(): # without dry-run result = runner.invoke( - export, [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date"] + export, [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date"] ) assert result.exit_code == 0 @@ -2675,7 +2742,7 @@ def test_export_touch_files_update(): # --update result = runner.invoke( export, - [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"], ) assert result.exit_code == 0 @@ -2688,7 +2755,7 @@ def test_export_touch_files_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2702,15 +2769,15 @@ def test_export_touch_files_update(): in result.output ) - for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + for fname, mtime in zip(CLI_EXPORT_BY_DATE_NEED_TOUCH, CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) != mtime + assert int(st.st_mtime) != int(mtime) # --update --touch-file result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2723,9 +2790,9 @@ def test_export_touch_files_update(): in result.output ) - for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): + for fname, mtime in zip(CLI_EXPORT_BY_DATE_NEED_TOUCH, CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) == mtime + assert int(st.st_mtime) == int(mtime) # touch one file and run update again ts = time.time() @@ -2734,7 +2801,7 @@ def test_export_touch_files_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2749,12 +2816,12 @@ def test_export_touch_files_update(): for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) == mtime + assert int(st.st_mtime) == int(mtime) # run update without --touch-file result = runner.invoke( export, - [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"], ) assert result.exit_code == 0 @@ -2776,7 +2843,9 @@ def test_export_touch_files_exiftool_update(): os.environ["TZ"] = "US/Pacific" time.tzset() - + + setup_touch_tests() + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager @@ -2784,7 +2853,7 @@ def test_export_touch_files_exiftool_update(): # basic export with dry-run result = runner.invoke( export, - [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--dry-run"], + [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--dry-run"], ) assert result.exit_code == 0 @@ -2794,7 +2863,7 @@ def test_export_touch_files_exiftool_update(): # without dry-run result = runner.invoke( - export, [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date"] + export, [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date"] ) assert result.exit_code == 0 @@ -2805,7 +2874,7 @@ def test_export_touch_files_exiftool_update(): # --update result = runner.invoke( export, - [os.path.join(cwd, PHOTOS_DB_15_6), ".", "--export-by-date", "--update"], + [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"], ) assert result.exit_code == 0 @@ -2818,7 +2887,7 @@ def test_export_touch_files_exiftool_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2837,7 +2906,7 @@ def test_export_touch_files_exiftool_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2855,7 +2924,7 @@ def test_export_touch_files_exiftool_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2872,13 +2941,13 @@ def test_export_touch_files_exiftool_update(): for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) != mtime + assert int(st.st_mtime) != int(mtime) # --update --touch-file --exiftool result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2894,7 +2963,7 @@ def test_export_touch_files_exiftool_update(): for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) == mtime + assert int(st.st_mtime) == int(mtime) # touch one file and run update again ts = time.time() @@ -2903,7 +2972,7 @@ def test_export_touch_files_exiftool_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2919,13 +2988,13 @@ def test_export_touch_files_exiftool_update(): for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES): st = os.stat(fname) - assert int(st.st_mtime) == mtime + assert int(st.st_mtime) == int(mtime) # run --update --exiftool --touch-file again result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update", @@ -2943,7 +3012,7 @@ def test_export_touch_files_exiftool_update(): result = runner.invoke( export, [ - os.path.join(cwd, PHOTOS_DB_15_6), + os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--exiftool",