Merge pull request #207 from RhetTbull/issue206

Closes issue #206, adds --touch-file
This commit is contained in:
Rhet Turnbull
2020-08-23 11:18:31 -07:00
committed by GitHub
7 changed files with 646 additions and 73 deletions

View File

@@ -203,14 +203,14 @@ Options:
both images and movies). both images and movies).
--only-photos Search only for photos/images (default --only-photos Search only for photos/images (default
searches both images and movies). searches both images and movies).
--from-date [%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S] --from-date DATETIME Search by start item date, e.g.
Search by start item date, e.g. 2000-01-12T12:00:00,
2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 2001-01-12T12:00:00-07:00, or 2000-12-31
w/o TZ). (ISO 8601).
--to-date [%Y-%m-%d|%Y-%m-%dT%H:%M:%S|%Y-%m-%d %H:%M:%S] --to-date DATETIME Search by end item date, e.g.
Search by end item date, e.g. 2000-01-12T12:00:00,
2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 2001-01-12T12:00:00-07:00, or 2000-12-31
w/o TZ). (ISO 8601).
--deleted Include photos from the 'Recently Deleted' --deleted Include photos from the 'Recently Deleted'
folder. folder.
--deleted-only Include only photos from the 'Recently --deleted-only Include only photos from the 'Recently
@@ -222,6 +222,8 @@ Options:
--export-as-hardlink Hardlink files instead of copying them. --export-as-hardlink Hardlink files instead of copying them.
Cannot be used with --exiftool which creates Cannot be used with --exiftool which creates
copies of the files with embedded EXIF data. copies of the files with embedded EXIF data.
--touch-file Sets the file's modification time to match
photo date.
--overwrite Overwrite existing files. Default behavior --overwrite Overwrite existing files. Default behavior
is to add (1), (2), etc to filename if file is to add (1), (2), etc to filename if file
already exists. Use this with caution as it already exists. Use this with caution as it

View File

@@ -1124,7 +1124,7 @@ def query(
@click.option( @click.option(
"--touch-file", "--touch-file",
is_flag=True, is_flag=True,
help="Sets the file's modification time upon photo date.", help="Sets the file's modification time to match photo date.",
) )
@click.option( @click.option(
"--overwrite", "--overwrite",
@@ -1269,6 +1269,13 @@ def query(
"to a filesystem that doesn't support Mac OS extended attributes. Only use this if you get " "to a filesystem that doesn't support Mac OS extended attributes. Only use this if you get "
"an error while exporting.", "an error while exporting.",
) )
@click.option(
"--use-photos-export",
is_flag=True,
default=False,
hidden=True,
help="Force the use of AppleScript to export even if not missing (see also --download-missing).",
)
@DB_ARGUMENT @DB_ARGUMENT
@click.argument("dest", nargs=1, type=click.Path(exists=True)) @click.argument("dest", nargs=1, type=click.Path(exists=True))
@click.pass_obj @click.pass_obj
@@ -1350,6 +1357,7 @@ def export(
label, label,
deleted, deleted,
deleted_only, deleted_only,
use_photos_export,
): ):
""" Export photos from the Photos database. """ Export photos from the Photos database.
Export path DEST is required. Export path DEST is required.
@@ -1520,7 +1528,6 @@ def export(
deleted_only=deleted_only, deleted_only=deleted_only,
) )
results_exported = []
if photos: if photos:
if export_bursts: if export_bursts:
# add the burst_photos to the export set # add the burst_photos to the export set
@@ -1539,10 +1546,12 @@ def export(
# because the original code used --original-name as an option # because the original code used --original-name as an option
original_name = not current_name original_name = not current_name
results_exported = []
results_new = [] results_new = []
results_updated = [] results_updated = []
results_skipped = [] results_skipped = []
results_exif_updated = [] results_exif_updated = []
results_touched = []
if verbose_: if verbose_:
for p in photos: for p in photos:
results = export_photo( results = export_photo(
@@ -1572,12 +1581,14 @@ def export(
dry_run=dry_run, dry_run=dry_run,
touch_file=touch_file, touch_file=touch_file,
edited_suffix=edited_suffix, edited_suffix=edited_suffix,
use_photos_export=use_photos_export,
) )
results_exported.extend(results.exported) results_exported.extend(results.exported)
results_new.extend(results.new) results_new.extend(results.new)
results_updated.extend(results.updated) results_updated.extend(results.updated)
results_skipped.extend(results.skipped) results_skipped.extend(results.skipped)
results_exif_updated.extend(results.exif_updated) results_exif_updated.extend(results.exif_updated)
results_touched.extend(results.touched)
else: else:
# show progress bar # show progress bar
@@ -1610,30 +1621,40 @@ def export(
dry_run=dry_run, dry_run=dry_run,
touch_file=touch_file, touch_file=touch_file,
edited_suffix=edited_suffix, edited_suffix=edited_suffix,
use_photos_export=use_photos_export,
) )
results_exported.extend(results.exported) results_exported.extend(results.exported)
results_new.extend(results.new) results_new.extend(results.new)
results_updated.extend(results.updated) results_updated.extend(results.updated)
results_skipped.extend(results.skipped) results_skipped.extend(results.skipped)
results_exif_updated.extend(results.exif_updated) results_exif_updated.extend(results.exif_updated)
results_touched.extend(results.touched)
stop_time = time.perf_counter() stop_time = time.perf_counter()
# print summary results # print summary results
if update: if update:
photo_str_new = "photos" if len(results_new) != 1 else "photo" photo_str_new = "photos" if len(results_new) != 1 else "photo"
photo_str_updated = "photos" if len(results_new) != 1 else "photo" photo_str_updated = "photos" if len(results_updated) != 1 else "photo"
photo_str_skipped = "photos" if len(results_skipped) != 1 else "photo" photo_str_skipped = "photos" if len(results_skipped) != 1 else "photo"
photo_str_touched = "photos" if len(results_touched) != 1 else "photo"
photo_str_exif_updated = ( photo_str_exif_updated = (
"photos" if len(results_exif_updated) != 1 else "photo" "photos" if len(results_exif_updated) != 1 else "photo"
) )
click.echo( summary = (
f"Exported: {len(results_new)} {photo_str_new}, " f"Exported: {len(results_new)} {photo_str_new}, "
+ f"updated: {len(results_updated)} {photo_str_updated}, " f"updated: {len(results_updated)} {photo_str_updated}, "
+ f"skipped: {len(results_skipped)} {photo_str_skipped}, " f"skipped: {len(results_skipped)} {photo_str_skipped}, "
+ f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}" f"updated EXIF data: {len(results_exif_updated)} {photo_str_exif_updated}"
) )
if touch_file:
summary += f", touched date: {len(results_touched)} {photo_str_touched}"
click.echo(summary)
else: else:
photo_str = "photos" if len(results_exported) != 1 else "photo" photo_str = "photos" if len(results_exported) != 1 else "photo"
click.echo(f"Exported: {len(results_exported)} {photo_str}") photo_str_touched = "photos" if len(results_touched) != 1 else "photo"
summary = f"Exported: {len(results_exported)} {photo_str}"
if touch_file:
summary += f", touched date: {len(results_touched)} {photo_str_touched}"
click.echo(summary)
click.echo(f"Elapsed time: {(stop_time-start_time):.3f} seconds") click.echo(f"Elapsed time: {(stop_time-start_time):.3f} seconds")
else: else:
click.echo("Did not find any photos to export") click.echo("Did not find any photos to export")
@@ -2083,6 +2104,7 @@ def export_photo(
dry_run=None, dry_run=None,
touch_file=None, touch_file=None,
edited_suffix="_edited", edited_suffix="_edited",
use_photos_export=False,
): ):
""" Helper function for export that does the actual export """ Helper function for export that does the actual export
@@ -2110,7 +2132,8 @@ def export_photo(
export_db: export database instance compatible with ExportDB_ABC export_db: export database instance compatible with ExportDB_ABC
fileutil: file util class compatible with FileUtilABC fileutil: file util class compatible with FileUtilABC
dry_run: boolean; if True, doesn't actually export or update any files dry_run: boolean; if True, doesn't actually export or update any files
touch_file: boolean; sets file's modification time upon photo date touch_file: boolean; sets file's modification time to match photo date
use_photos_export: boolean; if True forces the use of AppleScript to export even if photo not missing
Returns: Returns:
list of path(s) of exported photo or None if photo was missing list of path(s) of exported photo or None if photo was missing
@@ -2125,25 +2148,26 @@ def export_photo(
if photo.ismissing: if photo.ismissing:
space = " " if not verbose_ else "" space = " " if not verbose_ else ""
verbose(f"{space}Skipping missing photo {photo.filename}") verbose(f"{space}Skipping missing photo {photo.filename}")
return ExportResults([], [], [], [], []) return ExportResults([], [], [], [], [], [])
elif not os.path.exists(photo.path): elif not os.path.exists(photo.path):
space = " " if not verbose_ else "" space = " " if not verbose_ else ""
verbose( verbose(
f"{space}WARNING: file {photo.path} is missing but ismissing=False, " f"{space}WARNING: file {photo.path} is missing but ismissing=False, "
f"skipping {photo.filename}" f"skipping {photo.filename}"
) )
return ExportResults([], [], [], [], []) return ExportResults([], [], [], [], [], [])
elif photo.ismissing and not photo.iscloudasset or not photo.incloud: elif photo.ismissing and not photo.iscloudasset or not photo.incloud:
verbose( verbose(
f"Skipping missing {photo.filename}: not iCloud asset or missing from cloud" f"Skipping missing {photo.filename}: not iCloud asset or missing from cloud"
) )
return ExportResults([], [], [], [], []) return ExportResults([], [], [], [], [], [])
results_exported = [] results_exported = []
results_new = [] results_new = []
results_updated = [] results_updated = []
results_skipped = [] results_skipped = []
results_exif_updated = [] results_exif_updated = []
results_touched = []
filenames = get_filenames_from_template(photo, filename_template, original_name) filenames = get_filenames_from_template(photo, filename_template, original_name)
for filename in filenames: for filename in filenames:
@@ -2162,8 +2186,10 @@ def export_photo(
# if download_missing and the photo is missing or path doesn't exist, # if download_missing and the photo is missing or path doesn't exist,
# try to download with Photos # try to download with Photos
use_photos_export = download_missing and ( use_photos_export = (
photo.ismissing or not os.path.exists(photo.path) download_missing and (photo.ismissing or not os.path.exists(photo.path))
if not use_photos_export
else True
) )
# export the photo to each path in dest_paths # export the photo to each path in dest_paths
@@ -2196,6 +2222,7 @@ def export_photo(
results_updated.extend(export_results.updated) results_updated.extend(export_results.updated)
results_skipped.extend(export_results.skipped) results_skipped.extend(export_results.skipped)
results_exif_updated.extend(export_results.exif_updated) results_exif_updated.extend(export_results.exif_updated)
results_touched.extend(export_results.touched)
if verbose_: if verbose_:
for exported in export_results.exported: for exported in export_results.exported:
@@ -2206,13 +2233,14 @@ def export_photo(
verbose(f"Exported updated file {updated}") verbose(f"Exported updated file {updated}")
for skipped in export_results.skipped: for skipped in export_results.skipped:
verbose(f"Skipped up to date file {skipped}") verbose(f"Skipped up to date file {skipped}")
for touched in export_results.touched:
verbose(f"Touched date on file {touched}")
# if export-edited, also export the edited version # if export-edited, also export the edited version
# verify the photo has adjustments and valid path to avoid raising an exception # verify the photo has adjustments and valid path to avoid raising an exception
if export_edited and photo.hasadjustments: if export_edited and photo.hasadjustments:
# if download_missing and the photo is missing or path doesn't exist, # if download_missing and the photo is missing or path doesn't exist,
# try to download with Photos # try to download with Photos
use_photos_export = download_missing and photo.path_edited is None
if not download_missing and photo.path_edited is None: if not download_missing and photo.path_edited is None:
verbose(f"Skipping missing edited photo for {filename}") verbose(f"Skipping missing edited photo for {filename}")
else: else:
@@ -2253,6 +2281,7 @@ def export_photo(
results_updated.extend(export_results_edited.updated) results_updated.extend(export_results_edited.updated)
results_skipped.extend(export_results_edited.skipped) results_skipped.extend(export_results_edited.skipped)
results_exif_updated.extend(export_results_edited.exif_updated) results_exif_updated.extend(export_results_edited.exif_updated)
results_touched.extend(export_results_edited.touched)
if verbose_: if verbose_:
for exported in export_results_edited.exported: for exported in export_results_edited.exported:
@@ -2263,6 +2292,8 @@ def export_photo(
verbose(f"Exported updated file {updated}") verbose(f"Exported updated file {updated}")
for skipped in export_results_edited.skipped: for skipped in export_results_edited.skipped:
verbose(f"Skipped up to date file {skipped}") verbose(f"Skipped up to date file {skipped}")
for touched in export_results_edited.touched:
verbose(f"Touched date on file {touched}")
return ExportResults( return ExportResults(
results_exported, results_exported,
@@ -2270,6 +2301,7 @@ def export_photo(
results_updated, results_updated,
results_skipped, results_skipped,
results_exif_updated, results_exif_updated,
results_touched,
) )

View File

@@ -189,7 +189,12 @@ class ExportDB(ExportDB_ABC):
(filename,), (filename,),
) )
results = c.fetchone() results = c.fetchone()
stats = results[0:3] if results else None if results:
stats = results[0:3]
mtime = int(stats[2]) if stats[2] is not None else None
stats = (stats[0], stats[1], mtime)
else:
stats = (None, None, None)
except Error as e: except Error as e:
logging.warning(e) logging.warning(e)
stats = (None, None, None) stats = (None, None, None)
@@ -232,7 +237,12 @@ class ExportDB(ExportDB_ABC):
(filename,), (filename,),
) )
results = c.fetchone() results = c.fetchone()
stats = results[0:3] if results else None if results:
stats = results[0:3]
mtime = int(stats[2]) if stats[2] is not None else None
stats = (stats[0], stats[1], mtime)
else:
stats = (None, None, None)
except Error as e: except Error as e:
logging.warning(e) logging.warning(e)
stats = (None, None, None) stats = (None, None, None)

View File

@@ -1,3 +1,3 @@
""" version info """ """ version info """
__version__ = "0.33.1" __version__ = "0.33.2"

View File

@@ -34,7 +34,12 @@ class FileUtilABC(ABC):
@classmethod @classmethod
@abstractmethod @abstractmethod
def cmp_sig(cls, file1, file2): def cmp(cls, file1, file2, mtime1=None):
pass
@classmethod
@abstractmethod
def cmp_file_sig(cls, file1, file2):
pass pass
@classmethod @classmethod
@@ -114,11 +119,32 @@ class FileUtilMacOS(FileUtilABC):
os.utime(path, times) os.utime(path, times)
@classmethod @classmethod
def cmp_sig(cls, f1, s2): def cmp(cls, f1, f2, mtime1=None):
"""Does shallow compare (file signatures) of f1 to file f2.
Arguments:
f1 -- File name
f2 -- File name
mtime1 -- optional, pass alternate file modification timestamp for f1; will be converted to int
Return value:
True if the file signatures as returned by stat are the same, False otherwise.
Does not do a byte-by-byte comparison.
"""
s1 = cls._sig(os.stat(f1))
if mtime1 is not None:
s1 = (s1[0], s1[1], int(mtime1))
s2 = cls._sig(os.stat(f2))
if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
return False
return s1 == s2
@classmethod
def cmp_file_sig(cls, f1, s2):
"""Compare file f1 to signature s2. """Compare file f1 to signature s2.
Arguments: Arguments:
f1 -- File name f1 -- File name
s2 -- stats as returned by sig s2 -- stats as returned by _sig
Return value: Return value:
True if the files are the same, False otherwise. True if the files are the same, False otherwise.
@@ -140,7 +166,12 @@ class FileUtilMacOS(FileUtilABC):
@staticmethod @staticmethod
def _sig(st): def _sig(st):
return (stat.S_IFMT(st.st_mode), st.st_size, st.st_mtime) """ return tuple of (mode, size, mtime) of file based on os.stat
Args:
st: os.stat signature
"""
# use int(st.st_mtime) because ditto does not copy fractional portion of mtime
return (stat.S_IFMT(st.st_mode), st.st_size, int(st.st_mtime))
class FileUtil(FileUtilMacOS): class FileUtil(FileUtilMacOS):
@@ -151,8 +182,8 @@ class FileUtil(FileUtilMacOS):
class FileUtilNoOp(FileUtil): class FileUtilNoOp(FileUtil):
""" No-Op implementation of FileUtil for testing / dry-run mode """ No-Op implementation of FileUtil for testing / dry-run mode
all methods with exception of cmp_sig and file_cmp are no-op all methods with exception of cmp, cmp_file_sig and file_cmp are no-op
cmp_sig functions as FileUtil.cmp_sig does cmp and cmp_file_sig functions as FileUtil methods do
file_cmp returns mock data file_cmp returns mock data
""" """

View File

@@ -11,7 +11,6 @@
# TODO: should this be its own PhotoExporter class? # TODO: should this be its own PhotoExporter class?
import filecmp
import glob import glob
import json import json
import logging import logging
@@ -37,7 +36,8 @@ from ..fileutil import FileUtil
from ..utils import dd_to_dms_str, findfiles from ..utils import dd_to_dms_str, findfiles
ExportResults = namedtuple( ExportResults = namedtuple(
"ExportResults", ["exported", "new", "updated", "skipped", "exif_updated"] "ExportResults",
["exported", "new", "updated", "skipped", "exif_updated", "touched"],
) )
@@ -377,6 +377,9 @@ def export2(
# list of all files skipped because they do not need to be updated (for use with update=True) # list of all files skipped because they do not need to be updated (for use with update=True)
update_skipped_files = [] update_skipped_files = []
# list of all files with utime touched (touch_file = True)
touched_files = []
# check edited and raise exception trying to export edited version of # check edited and raise exception trying to export edited version of
# photo that hasn't been edited # photo that hasn't been edited
if edited and not self.hasadjustments: if edited and not self.hasadjustments:
@@ -484,7 +487,7 @@ def export2(
if update and dest.exists(): if update and dest.exists():
# destination exists, check to see if destination is the right UUID # destination exists, check to see if destination is the right UUID
dest_uuid = export_db.get_uuid_for_file(dest) dest_uuid = export_db.get_uuid_for_file(dest)
if dest_uuid is None and filecmp.cmp(src, dest): if dest_uuid is None and fileutil.cmp(src, dest):
# might be exporting into a pre-ExportDB folder or the DB got deleted # might be exporting into a pre-ExportDB folder or the DB got deleted
logging.debug( logging.debug(
f"Found matching file with blank uuid: {self.uuid}, {dest}" f"Found matching file with blank uuid: {self.uuid}, {dest}"
@@ -516,7 +519,7 @@ def export2(
dest = pathlib.Path(file_) dest = pathlib.Path(file_)
found_match = True found_match = True
break break
elif dest_uuid is None and filecmp.cmp(src, file_): elif dest_uuid is None and fileutil.cmp(src, file_):
# files match, update the UUID # files match, update the UUID
logging.debug( logging.debug(
f"Found matching file with blank uuid: {self.uuid}, {file_}" f"Found matching file with blank uuid: {self.uuid}, {file_}"
@@ -567,6 +570,7 @@ def export2(
update_new_files = results.new update_new_files = results.new
update_updated_files = results.updated update_updated_files = results.updated
update_skipped_files = results.skipped update_skipped_files = results.skipped
touched_files = results.touched
# copy live photo associated .mov if requested # copy live photo associated .mov if requested
if live_photo and self.live_photo: if live_photo and self.live_photo:
@@ -593,6 +597,7 @@ def export2(
update_new_files.extend(results.new) update_new_files.extend(results.new)
update_updated_files.extend(results.updated) update_updated_files.extend(results.updated)
update_skipped_files.extend(results.skipped) update_skipped_files.extend(results.skipped)
touched_files.extend(results.touched)
else: else:
logging.debug(f"Skipping missing live movie for {filename}") logging.debug(f"Skipping missing live movie for {filename}")
@@ -619,11 +624,12 @@ def export2(
update_new_files.extend(results.new) update_new_files.extend(results.new)
update_updated_files.extend(results.updated) update_updated_files.extend(results.updated)
update_skipped_files.extend(results.skipped) update_skipped_files.extend(results.skipped)
touched_files.extend(results.touched)
else: else:
logging.debug(f"Skipping missing RAW photo for {filename}") logging.debug(f"Skipping missing RAW photo for {filename}")
else: else:
# use_photo_export # use_photo_export
exported = None exported = []
# export live_photo .mov file? # export live_photo .mov file?
live_photo = True if live_photo and self.live_photo else False live_photo = True if live_photo and self.live_photo else False
if edited: if edited:
@@ -633,7 +639,7 @@ def export2(
filestem = dest.stem filestem = dest.stem
else: else:
# didn't get passed a filename, add _edited # didn't get passed a filename, add _edited
filestem = f"{dest.stem}_edited" filestem = f"{dest.stem}{edited_identifier}"
dest = dest.parent / f"{filestem}.jpeg" dest = dest.parent / f"{filestem}.jpeg"
exported = _export_photo_uuid_applescript( exported = _export_photo_uuid_applescript(
@@ -662,8 +668,16 @@ def export2(
dry_run=dry_run, dry_run=dry_run,
) )
if exported is not None: if exported:
if touch_file:
for exported_file in exported:
touched_files.append(exported_file)
ts = int(self.date.timestamp())
fileutil.utime(exported_file, (ts, ts))
exported_files.extend(exported) exported_files.extend(exported)
if update:
update_new_files.extend(exported)
else: else:
logging.warning( logging.warning(
f"Error exporting photo {self.uuid} to {dest} with use_photos_export" f"Error exporting photo {self.uuid} to {dest} with use_photos_export"
@@ -765,6 +779,7 @@ def export2(
keyword_template=keyword_template, keyword_template=keyword_template,
description_template=description_template, description_template=description_template,
) )
export_db.set_exifdata_for_file( export_db.set_exifdata_for_file(
exported_file, exported_file,
self._exiftool_json_sidecar( self._exiftool_json_sidecar(
@@ -779,13 +794,23 @@ def export2(
) )
exif_files_updated.append(exported_file) exif_files_updated.append(exported_file)
return ExportResults( if touch_file:
for exif_file in exif_files_updated:
touched_files.append(exported_file)
ts = int(self.date.timestamp())
fileutil.utime(exported_file, (ts, ts))
touched_files = list(set(touched_files))
results = ExportResults(
exported_files, exported_files,
update_new_files, update_new_files,
update_updated_files, update_updated_files,
update_skipped_files, update_skipped_files,
exif_files_updated, exif_files_updated,
touched_files,
) )
return results
def _export_photo( def _export_photo(
@@ -827,6 +852,7 @@ def _export_photo(
update_updated_files = [] update_updated_files = []
update_new_files = [] update_new_files = []
update_skipped_files = [] update_skipped_files = []
touched_files = []
dest_str = str(dest) dest_str = str(dest)
dest_exists = dest.exists() dest_exists = dest.exists()
@@ -836,57 +862,89 @@ def _export_photo(
op_desc = "export_by_copying" op_desc = "export_by_copying"
if not update: if not update:
# not update, do the the hardlink # not update, export the file
logging.debug(f"Not update: {op_desc} linking file {src} {dest}") logging.debug(f"Exporting file with {op_desc} {src} {dest}")
exported_files.append(dest_str) exported_files.append(dest_str)
else: #updating if touch_file:
sig = fileutil.file_sig(src)
sig = (sig[0], sig[1], int(self.date.timestamp()))
if not fileutil.cmp_file_sig(src, sig):
touched_files.append(dest_str)
else: # updating
if not dest_exists: if not dest_exists:
# update, destination doesn't exist (new file) # update, destination doesn't exist (new file)
logging.debug(f"Update: exporting new file with {op_desc} {src} {dest}") logging.debug(f"Update: exporting new file with {op_desc} {src} {dest}")
update_new_files.append(dest_str) update_new_files.append(dest_str)
if touch_file:
touched_files.append(dest_str)
else: else:
# update, destination exists, but we might not need to replace it... # update, destination exists, but we might not need to replace it...
if (( export_as_hardlink and dest.samefile(src)) or if exiftool:
(not export_as_hardlink and not dest.samefile(src) and ( sig_exif = export_db.get_stat_exif_for_file(dest_str)
( exiftool and fileutil.cmp_sig(dest_str, export_db.get_stat_exif_for_file(dest_str))) or cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif)
(not exiftool and filecmp.cmp(src, dest))) sig_exif = (sig_exif[0], sig_exif[1], int(self.date.timestamp()))
)): cmp_touch = fileutil.cmp_file_sig(dest_str, sig_exif)
# destination exists but its signature is "identical" else:
logging.debug(f"Update: skipping identical original files {src} {dest}") cmp_orig = fileutil.cmp(src, dest)
# call set_stat because code can reach this spot if no export DB but exporting a RAW or live photo cmp_touch = fileutil.cmp(src, dest, mtime1=int(self.date.timestamp()))
# potentially re-writes the data in the database but ensures database is complete
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str)) sig_cmp = cmp_touch if touch_file else cmp_orig
if (export_as_hardlink and dest.samefile(src)) or (
not export_as_hardlink and not dest.samefile(src) and sig_cmp
):
# destination exists and signatures match, skip it
update_skipped_files.append(dest_str) update_skipped_files.append(dest_str)
else: else:
# destination exists but is different # destination exists but signature is different
logging.debug(f"Update: removing existing file prior to {op_desc} {src} {dest}") if touch_file and cmp_orig and not cmp_touch:
update_updated_files.append(dest_str) # destination exists, signature matches original but does not match expected touch time
# skip exporting but update touch time
update_skipped_files.append(dest_str)
touched_files.append(dest_str)
elif not touch_file and cmp_touch and not cmp_orig:
# destination exists, signature matches expected touch but not original
# user likely exported with touch_file and is now exporting without touch_file
# don't update the file because it's same but leave touch time
update_skipped_files.append(dest_str)
else:
# destination exists but is different
update_updated_files.append(dest_str)
if touch_file:
touched_files.append(dest_str)
if not update_skipped_files: if not update_skipped_files:
if dest_exists and (update or overwrite): if dest_exists and (update or overwrite):
# need to remove the destination first # need to remove the destination first
logging.debug(f"Update: removing existing file prior to export_as_hardlink {src} {dest}") logging.debug(
# dest.unlink() f"Update: removing existing file prior to {op_desc} {src} {dest}"
)
fileutil.unlink(dest) fileutil.unlink(dest)
if export_as_hardlink: if export_as_hardlink:
fileutil.hardlink(src, dest) fileutil.hardlink(src, dest)
else: else:
fileutil.copy(src, dest_str, norsrc=no_xattr) fileutil.copy(src, dest_str, norsrc=no_xattr)
if touch_file:
ts=self.date.timestamp()
fileutil.utime(dest, (ts, ts))
export_db.set_data( export_db.set_data(
dest_str, dest_str,
self.uuid, self.uuid,
fileutil.file_sig(dest_str), fileutil.file_sig(dest_str),
(None, None, None), (None, None, None),
self.json(), self.json(),
None, None,
) )
if touched_files:
ts = int(self.date.timestamp())
fileutil.utime(dest, (ts, ts))
return ExportResults( return ExportResults(
exported_files + update_new_files + update_updated_files, update_new_files, update_updated_files, update_skipped_files, [] exported_files + update_new_files + update_updated_files,
update_new_files,
update_updated_files,
update_skipped_files,
[],
touched_files,
) )

View File

@@ -14,6 +14,8 @@ PLACES_PHOTOS_DB = "tests/Test-Places-Catalina-10_15_1.photoslibrary"
PLACES_PHOTOS_DB_13 = "tests/Test-Places-High-Sierra-10.13.6.photoslibrary" PLACES_PHOTOS_DB_13 = "tests/Test-Places-High-Sierra-10.13.6.photoslibrary"
PHOTOS_DB_15_4 = "tests/Test-10.15.4.photoslibrary" PHOTOS_DB_15_4 = "tests/Test-10.15.4.photoslibrary"
PHOTOS_DB_15_5 = "tests/Test-10.15.5.photoslibrary" PHOTOS_DB_15_5 = "tests/Test-10.15.5.photoslibrary"
PHOTOS_DB_15_6 = "tests/Test-10.15.6.photoslibrary"
PHOTOS_DB_TOUCH = PHOTOS_DB_15_6
PHOTOS_DB_14_6 = "tests/Test-10.14.6.photoslibrary" PHOTOS_DB_14_6 = "tests/Test-10.14.6.photoslibrary"
UUID_FILE = "tests/uuid_from_file.txt" UUID_FILE = "tests/uuid_from_file.txt"
@@ -182,6 +184,20 @@ CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B"
CLI_EXPORT_UUID_FILENAME = "Pumkins2.jpg" CLI_EXPORT_UUID_FILENAME = "Pumkins2.jpg"
CLI_EXPORT_BY_DATE_TOUCH_UUID = [
"1EB2B765-0765-43BA-A90C-0D0580E6172C",
"F12384F6-CD17-4151-ACBA-AE0E3688539E",
]
CLI_EXPORT_BY_DATE_TOUCH_TIMES = [1538165373, 1538163349]
CLI_EXPORT_BY_DATE_NEED_TOUCH = [
"2018/09/28/Pumkins2.jpg",
"2018/10/13/St James Park.jpg",
]
CLI_EXPORT_BY_DATE_NEED_TOUCH_UUID = [
"D79B8D77-BFFC-460B-9312-034F2877D35B",
"DC99FBDD-7A52-4100-A5BB-344131646C30",
]
CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES = [1538165227, 1539436692]
CLI_EXPORT_BY_DATE = ["2018/09/28/Pumpkins3.jpg", "2018/09/28/Pumkins1.jpg"] CLI_EXPORT_BY_DATE = ["2018/09/28/Pumpkins3.jpg", "2018/09/28/Pumkins1.jpg"]
CLI_EXPORT_SIDECAR_FILENAMES = ["Pumkins2.jpg", "Pumkins2.json", "Pumkins2.xmp"] CLI_EXPORT_SIDECAR_FILENAMES = ["Pumkins2.jpg", "Pumkins2.json", "Pumkins2.xmp"]
@@ -323,6 +339,55 @@ except:
exiftool = None exiftool = None
def touch_all_photos_in_db(dbpath):
""" touch date on all photos in a library
helper function for --touch-file tests
Args:
dbpath: path to photos library to touch
"""
import os
import time
import osxphotos
ts = int(time.time())
for photo in osxphotos.PhotosDB(dbpath).photos():
if photo.path is not None:
os.utime(photo.path, (ts, ts))
if photo.path_edited is not None:
os.utime(photo.path_edited, (ts, ts))
if photo.path_raw is not None:
os.utime(photo.path_raw, (ts, ts))
if photo.path_live_photo is not None:
os.utime(photo.path_live_photo, (ts, ts))
def setup_touch_tests():
""" perform setup needed for --touch-file tests """
import os
import time
import logging
import osxphotos
touch_all_photos_in_db(PHOTOS_DB_TOUCH)
photos = osxphotos.PhotosDB(PHOTOS_DB_TOUCH).photos_by_uuid(
CLI_EXPORT_BY_DATE_TOUCH_UUID
)
for photo in photos:
logging.warning(photo.path)
ts = int(photo.date.timestamp())
if photo.path is not None:
os.utime(photo.path, (ts, ts))
if photo.path_edited is not None:
os.utime(photo.path_edited, (ts, ts))
if photo.path_raw is not None:
os.utime(photo.path_raw, (ts, ts))
if photo.path_live_photo is not None:
os.utime(photo.path_live_photo, (ts, ts))
def test_osxphotos(): def test_osxphotos():
import osxphotos import osxphotos
from osxphotos.__main__ import cli from osxphotos.__main__ import cli
@@ -706,7 +771,7 @@ def test_query_date_1():
import time import time
from osxphotos.__main__ import query from osxphotos.__main__ import query
os.environ['TZ'] = "US/Pacific" os.environ["TZ"] = "US/Pacific"
time.tzset() time.tzset()
runner = CliRunner() runner = CliRunner()
@@ -726,6 +791,7 @@ def test_query_date_1():
json_got = json.loads(result.output) json_got = json.loads(result.output)
assert len(json_got) == 4 assert len(json_got) == 4
def test_query_date_2(): def test_query_date_2():
""" Test --from-date and --to-date """ """ Test --from-date and --to-date """
import json import json
@@ -735,7 +801,7 @@ def test_query_date_2():
import time import time
from osxphotos.__main__ import query from osxphotos.__main__ import query
os.environ['TZ'] = "Asia/Jerusalem" os.environ["TZ"] = "Asia/Jerusalem"
time.tzset() time.tzset()
runner = CliRunner() runner = CliRunner()
@@ -755,6 +821,7 @@ def test_query_date_2():
json_got = json.loads(result.output) json_got = json.loads(result.output)
assert len(json_got) == 2 assert len(json_got) == 2
def test_query_date_timezone(): def test_query_date_timezone():
""" Test --from-date, --to-date with ISO 8601 timezone """ """ Test --from-date, --to-date with ISO 8601 timezone """
import json import json
@@ -764,7 +831,7 @@ def test_query_date_timezone():
import time import time
from osxphotos.__main__ import query from osxphotos.__main__ import query
os.environ['TZ'] = "US/Pacific" os.environ["TZ"] = "US/Pacific"
time.tzset() time.tzset()
runner = CliRunner() runner = CliRunner()
@@ -2300,6 +2367,16 @@ def test_export_update_exiftool():
in result.output in result.output
) )
# update with exiftool again, should be no changes
result = runner.invoke(
export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "--update", "--exiftool"]
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 8 photos, updated EXIF data: 0 photos"
in result.output
)
def test_export_update_hardlink(): def test_export_update_hardlink():
""" test export with hardlink then update """ """ test export with hardlink then update """
@@ -2587,6 +2664,369 @@ def test_export_directory_template_1_dry_run():
assert not os.path.isfile(os.path.join(workdir, filepath)) assert not os.path.isfile(os.path.join(workdir, filepath))
def test_export_touch_files():
""" test export with --touch-files """
import os
import time
import osxphotos
from osxphotos.__main__ import export
os.environ["TZ"] = "US/Pacific"
time.tzset()
setup_touch_tests()
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"-V",
"--touch-file",
"--export-by-date",
],
)
assert result.exit_code == 0
assert "Exported: 16 photos, touched date: 14 photos" in result.output
for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) == int(mtime)
def test_export_touch_files_update():
""" test complex export scenario with --update and --touch-files """
import os
import pathlib
import time
import osxphotos
from osxphotos.__main__ import export
os.environ["TZ"] = "US/Pacific"
time.tzset()
setup_touch_tests()
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
# basic export with dry-run
result = runner.invoke(
export,
[os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--dry-run"],
)
assert result.exit_code == 0
assert "Exported: 16 photos" in result.output
assert not pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file()
# without dry-run
result = runner.invoke(
export, [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date"]
)
assert result.exit_code == 0
assert "Exported: 16 photos" in result.output
assert pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file()
# --update
result = runner.invoke(
export,
[os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos"
in result.output
)
# --update --touch-file --dry-run
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--touch-file",
"--dry-run",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 14 photos"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE_NEED_TOUCH, CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) != int(mtime)
# --update --touch-file
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--touch-file",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 14 photos"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE_NEED_TOUCH, CLI_EXPORT_BY_DATE_NEED_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) == int(mtime)
# touch one file and run update again
ts = time.time()
os.utime(CLI_EXPORT_BY_DATE[0], (ts, ts))
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--touch-file",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 1 photo, skipped: 15 photos, updated EXIF data: 0 photos, touched date: 1 photo"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) == int(mtime)
# run update without --touch-file
result = runner.invoke(
export,
[os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos"
in result.output
)
@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
def test_export_touch_files_exiftool_update():
""" test complex export scenario with --update, --exiftol, and --touch-files """
import os
import pathlib
import time
import osxphotos
from osxphotos.__main__ import export
os.environ["TZ"] = "US/Pacific"
time.tzset()
setup_touch_tests()
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
# basic export with dry-run
result = runner.invoke(
export,
[os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--dry-run"],
)
assert result.exit_code == 0
assert "Exported: 16 photos" in result.output
assert not pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file()
# without dry-run
result = runner.invoke(
export, [os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date"]
)
assert result.exit_code == 0
assert "Exported: 16 photos" in result.output
assert pathlib.Path(CLI_EXPORT_BY_DATE[0]).is_file()
# --update
result = runner.invoke(
export,
[os.path.join(cwd, PHOTOS_DB_TOUCH), ".", "--export-by-date", "--update"],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos"
in result.output
)
# --update --exiftool --dry-run
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
"--dry-run",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 16 photos, skipped: 0 photos, updated EXIF data: 16 photos"
in result.output
)
# --update --exiftool
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 16 photos, skipped: 0 photos, updated EXIF data: 16 photos"
in result.output
)
# --update --touch-file --exiftool --dry-run
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
"--touch-file",
"--dry-run",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 16 photos"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) != int(mtime)
# --update --touch-file --exiftool
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
"--touch-file",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 16 photos"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) == int(mtime)
# touch one file and run update again
ts = time.time()
os.utime(CLI_EXPORT_BY_DATE[0], (ts, ts))
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
"--touch-file",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 1 photo, skipped: 15 photos, updated EXIF data: 1 photo, touched date: 1 photo"
in result.output
)
for fname, mtime in zip(CLI_EXPORT_BY_DATE, CLI_EXPORT_BY_DATE_TOUCH_TIMES):
st = os.stat(fname)
assert int(st.st_mtime) == int(mtime)
# run --update --exiftool --touch-file again
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--update",
"--exiftool",
"--touch-file",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos, touched date: 0 photos"
in result.output
)
# run update without --touch-file
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_TOUCH),
".",
"--export-by-date",
"--exiftool",
"--update",
],
)
assert result.exit_code == 0
assert (
"Exported: 0 photos, updated: 0 photos, skipped: 16 photos, updated EXIF data: 0 photos"
in result.output
)
def test_labels(): def test_labels():
"""Test osxphotos labels """ """Test osxphotos labels """
import json import json