diff --git a/README.md b/README.md index 7d84771d..bd88f590 100644 --- a/README.md +++ b/README.md @@ -783,8 +783,15 @@ Options: folder. --deleted-only Include only photos from the 'Recently Deleted' folder. - --update Only export new or updated files. See notes - below on export and --update. + --update Only export new or updated files. See also + --force-update and notes below on export and + --update. + --force-update Only export new or updated files. Unlike + --update, --force-update will re-export photos + if their metadata has changed even if this + would not otherwise trigger an export. See + also --update and notes below on export and + --update. --ignore-signature When used with '--update', ignores file signature when updating files. This is useful if you have processed or edited exported @@ -1725,7 +1732,7 @@ Substitution Description {lf} A line feed: '\n', alias for {newline} {cr} A carriage return: '\r' {crlf} a carriage return + line feed: '\r\n' -{osxphotos_version} The osxphotos version, e.g. '0.45.8' +{osxphotos_version} The osxphotos version, e.g. '0.45.9' {osxphotos_cmd_line} The full command line used to run osxphotos The following substitutions may result in multiple values. Thus if specified for @@ -3629,7 +3636,7 @@ The following template field substitutions are availabe for use the templating s |{lf}|A line feed: '\n', alias for {newline}| |{cr}|A carriage return: '\r'| |{crlf}|a carriage return + line feed: '\r\n'| -|{osxphotos_version}|The osxphotos version, e.g. '0.45.8'| +|{osxphotos_version}|The osxphotos version, e.g. '0.45.9'| |{osxphotos_cmd_line}|The full command line used to run osxphotos| |{album}|Album(s) photo is contained in| |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder| diff --git a/docs/.buildinfo b/docs/.buildinfo index 9defd753..98e532c3 100644 --- a/docs/.buildinfo +++ b/docs/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: bf43bf49b725c31ce72a8823e4f8012b +config: a320d2e66b198895ef0b12b1f5934727 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_static/documentation_options.js b/docs/_static/documentation_options.js index 0896a973..4e9dfbd4 100644 --- a/docs/_static/documentation_options.js +++ b/docs/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '0.45.8', + VERSION: '0.45.9', LANGUAGE: 'None', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/docs/cli.html b/docs/cli.html index 1dc80919..80bf6c2b 100644 --- a/docs/cli.html +++ b/docs/cli.html @@ -6,7 +6,7 @@ - osxphotos command line interface (CLI) — osxphotos 0.45.8 documentation + osxphotos command line interface (CLI) — osxphotos 0.45.9 documentation diff --git a/docs/genindex.html b/docs/genindex.html index 0d697e09..d42baa7d 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -5,7 +5,7 @@ - Index — osxphotos 0.45.8 documentation + Index — osxphotos 0.45.9 documentation diff --git a/docs/index.html b/docs/index.html index f1d4cf16..ca087b89 100644 --- a/docs/index.html +++ b/docs/index.html @@ -6,7 +6,7 @@ - Welcome to osxphotos’s documentation! — osxphotos 0.45.8 documentation + Welcome to osxphotos’s documentation! — osxphotos 0.45.9 documentation diff --git a/docs/modules.html b/docs/modules.html index dd54b250..16224df4 100644 --- a/docs/modules.html +++ b/docs/modules.html @@ -6,7 +6,7 @@ - osxphotos — osxphotos 0.45.8 documentation + osxphotos — osxphotos 0.45.9 documentation diff --git a/docs/reference.html b/docs/reference.html index 20928185..4a3c30f4 100644 --- a/docs/reference.html +++ b/docs/reference.html @@ -6,7 +6,7 @@ - osxphotos package — osxphotos 0.45.8 documentation + osxphotos package — osxphotos 0.45.9 documentation diff --git a/docs/search.html b/docs/search.html index 7fa67509..8a4d69fa 100644 --- a/docs/search.html +++ b/docs/search.html @@ -5,7 +5,7 @@ - Search — osxphotos 0.45.8 documentation + Search — osxphotos 0.45.9 documentation diff --git a/osxphotos/_version.py b/osxphotos/_version.py index 267ed7ea..2a087d5e 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,3 +1,3 @@ """ version info """ -__version__ = "0.45.8" +__version__ = "0.45.9" diff --git a/osxphotos/cli.py b/osxphotos/cli.py index d8b5fc7a..76cc2e79 100644 --- a/osxphotos/cli.py +++ b/osxphotos/cli.py @@ -691,7 +691,15 @@ def cli(ctx, db, json_, debug): @click.option( "--update", is_flag=True, - help="Only export new or updated files. See notes below on export and --update.", + help="Only export new or updated files. " + "See also --force-update and notes below on export and --update.", +) +@click.option( + "--force-update", + is_flag=True, + help="Only export new or updated files. Unlike --update, --force-update will re-export photos " + "if their metadata has changed even if this would not otherwise trigger an export. " + "See also --update and notes below on export and --update.", ) @click.option( "--ignore-signature", @@ -1235,6 +1243,7 @@ def export( timestamp, missing, update, + force_update, ignore_signature, only_new, dry_run, @@ -1398,133 +1407,134 @@ def export( # re-set the local vars to the corresponding config value # this isn't elegant but avoids having to rewrite this function to use cfg.varname for every parameter - db = cfg.db - photos_library = cfg.photos_library - keyword = cfg.keyword - person = cfg.person + add_exported_to_album = cfg.add_exported_to_album + add_missing_to_album = cfg.add_missing_to_album + add_skipped_to_album = cfg.add_skipped_to_album album = cfg.album - folder = cfg.folder - name = cfg.name - uuid = cfg.uuid - uuid_from_file = cfg.uuid_from_file - title = cfg.title - no_title = cfg.no_title - description = cfg.description - no_description = cfg.no_description - uti = cfg.uti - ignore_case = cfg.ignore_case - edited = cfg.edited - external_edit = cfg.external_edit - favorite = cfg.favorite - not_favorite = cfg.not_favorite - hidden = cfg.hidden - not_hidden = cfg.not_hidden - shared = cfg.shared - not_shared = cfg.not_shared - from_date = cfg.from_date - to_date = cfg.to_date - from_time = cfg.from_time - to_time = cfg.to_time - verbose = cfg.verbose - missing = cfg.missing - update = cfg.update - ignore_signature = cfg.ignore_signature - dry_run = cfg.dry_run - export_as_hardlink = cfg.export_as_hardlink - touch_file = cfg.touch_file - overwrite = cfg.overwrite - retry = cfg.retry - export_by_date = cfg.export_by_date - skip_edited = cfg.skip_edited - skip_original_if_edited = cfg.skip_original_if_edited - skip_bursts = cfg.skip_bursts - skip_live = cfg.skip_live - skip_raw = cfg.skip_raw - skip_uuid = cfg.skip_uuid - skip_uuid_from_file = cfg.skip_uuid_from_file - person_keyword = cfg.person_keyword album_keyword = cfg.album_keyword - keyword_template = cfg.keyword_template - replace_keywords = cfg.replace_keywords - description_template = cfg.description_template - finder_tag_template = cfg.finder_tag_template - finder_tag_keywords = cfg.finder_tag_keywords - xattr_template = cfg.xattr_template - current_name = cfg.current_name - convert_to_jpeg = cfg.convert_to_jpeg - jpeg_quality = cfg.jpeg_quality - sidecar = cfg.sidecar - sidecar_drop_ext = cfg.sidecar_drop_ext - only_photos = cfg.only_photos - only_movies = cfg.only_movies + beta = cfg.beta burst = cfg.burst - not_burst = cfg.not_burst - live = cfg.live - not_live = cfg.not_live - download_missing = cfg.download_missing - exiftool = cfg.exiftool - exiftool_path = cfg.exiftool_path - exiftool_option = cfg.exiftool_option - exiftool_merge_keywords = cfg.exiftool_merge_keywords - exiftool_merge_persons = cfg.exiftool_merge_persons - ignore_date_modified = cfg.ignore_date_modified - portrait = cfg.portrait - not_portrait = cfg.not_portrait - screenshot = cfg.screenshot - not_screenshot = cfg.not_screenshot - slow_mo = cfg.slow_mo - not_slow_mo = cfg.not_slow_mo - time_lapse = cfg.time_lapse - not_time_lapse = cfg.not_time_lapse - hdr = cfg.hdr - not_hdr = cfg.not_hdr - selfie = cfg.selfie - not_selfie = cfg.not_selfie - panorama = cfg.panorama - not_panorama = cfg.not_panorama - has_raw = cfg.has_raw - directory = cfg.directory - filename_template = cfg.filename_template - jpeg_ext = cfg.jpeg_ext - strip = cfg.strip - edited_suffix = cfg.edited_suffix - original_suffix = cfg.original_suffix - place = cfg.place - no_place = cfg.no_place - location = cfg.location - no_location = cfg.no_location - has_comment = cfg.has_comment - no_comment = cfg.no_comment - has_likes = cfg.has_likes - no_likes = cfg.no_likes - label = cfg.label + cleanup = cfg.cleanup + convert_to_jpeg = cfg.convert_to_jpeg + current_name = cfg.current_name + db = cfg.db deleted = cfg.deleted deleted_only = cfg.deleted_only - use_photos_export = cfg.use_photos_export - use_photokit = cfg.use_photokit - report = cfg.report - cleanup = cfg.cleanup - add_exported_to_album = cfg.add_exported_to_album - add_skipped_to_album = cfg.add_skipped_to_album - add_missing_to_album = cfg.add_missing_to_album - exportdb = cfg.exportdb - beta = cfg.beta - only_new = cfg.only_new - in_album = cfg.in_album - not_in_album = cfg.not_in_album - min_size = cfg.min_size - max_size = cfg.max_size - regex = cfg.regex - selected = cfg.selected - exif = cfg.exif - query_eval = cfg.query_eval - query_function = cfg.query_function + description = cfg.description + description_template = cfg.description_template + directory = cfg.directory + download_missing = cfg.download_missing + dry_run = cfg.dry_run duplicate = cfg.duplicate + edited = cfg.edited + edited_suffix = cfg.edited_suffix + exif = cfg.exif + exiftool = cfg.exiftool + exiftool_merge_keywords = cfg.exiftool_merge_keywords + exiftool_merge_persons = cfg.exiftool_merge_persons + exiftool_option = cfg.exiftool_option + exiftool_path = cfg.exiftool_path + export_as_hardlink = cfg.export_as_hardlink + export_by_date = cfg.export_by_date + exportdb = cfg.exportdb + external_edit = cfg.external_edit + favorite = cfg.favorite + filename_template = cfg.filename_template + finder_tag_keywords = cfg.finder_tag_keywords + finder_tag_template = cfg.finder_tag_template + folder = cfg.folder + force_update = cfg.force_update + from_date = cfg.from_date + from_time = cfg.from_time + has_comment = cfg.has_comment + has_likes = cfg.has_likes + has_raw = cfg.has_raw + hdr = cfg.hdr + hidden = cfg.hidden + ignore_case = cfg.ignore_case + ignore_date_modified = cfg.ignore_date_modified + ignore_signature = cfg.ignore_signature + in_album = cfg.in_album + jpeg_ext = cfg.jpeg_ext + jpeg_quality = cfg.jpeg_quality + keyword = cfg.keyword + keyword_template = cfg.keyword_template + label = cfg.label + live = cfg.live + location = cfg.location + max_size = cfg.max_size + min_size = cfg.min_size + missing = cfg.missing + name = cfg.name + no_comment = cfg.no_comment + no_description = cfg.no_description + no_likes = cfg.no_likes + no_location = cfg.no_location + no_place = cfg.no_place + no_title = cfg.no_title + not_burst = cfg.not_burst + not_favorite = cfg.not_favorite + not_hdr = cfg.not_hdr + not_hidden = cfg.not_hidden + not_in_album = cfg.not_in_album + not_live = cfg.not_live + not_panorama = cfg.not_panorama + not_portrait = cfg.not_portrait + not_screenshot = cfg.not_screenshot + not_selfie = cfg.not_selfie + not_shared = cfg.not_shared + not_slow_mo = cfg.not_slow_mo + not_time_lapse = cfg.not_time_lapse + only_movies = cfg.only_movies + only_new = cfg.only_new + only_photos = cfg.only_photos + original_suffix = cfg.original_suffix + overwrite = cfg.overwrite + panorama = cfg.panorama + person = cfg.person + person_keyword = cfg.person_keyword + photos_library = cfg.photos_library + place = cfg.place + portrait = cfg.portrait post_command = cfg.post_command post_function = cfg.post_function preview = cfg.preview - preview_suffix = cfg.preview_suffix preview_if_missing = cfg.preview_if_missing + preview_suffix = cfg.preview_suffix + query_eval = cfg.query_eval + query_function = cfg.query_function + regex = cfg.regex + replace_keywords = cfg.replace_keywords + report = cfg.report + retry = cfg.retry + screenshot = cfg.screenshot + selected = cfg.selected + selfie = cfg.selfie + shared = cfg.shared + sidecar = cfg.sidecar + sidecar_drop_ext = cfg.sidecar_drop_ext + skip_bursts = cfg.skip_bursts + skip_edited = cfg.skip_edited + skip_live = cfg.skip_live + skip_original_if_edited = cfg.skip_original_if_edited + skip_raw = cfg.skip_raw + skip_uuid = cfg.skip_uuid + skip_uuid_from_file = cfg.skip_uuid_from_file + slow_mo = cfg.slow_mo + strip = cfg.strip + time_lapse = cfg.time_lapse + title = cfg.title + to_date = cfg.to_date + to_time = cfg.to_time + touch_file = cfg.touch_file + update = cfg.update + use_photokit = cfg.use_photokit + use_photos_export = cfg.use_photos_export + uti = cfg.uti + uuid = cfg.uuid + uuid_from_file = cfg.uuid_from_file + verbose = cfg.verbose + xattr_template = cfg.xattr_template # config file might have changed verbose VERBOSE = bool(verbose) @@ -1564,8 +1574,8 @@ def export( dependent_options = [ ("missing", ("download_missing", "use_photos_export")), ("jpeg_quality", ("convert_to_jpeg")), - ("ignore_signature", ("update")), - ("only_new", ("update")), + ("ignore_signature", ("update", "force_update")), + ("only_new", ("update", "force_update")), ("exiftool_option", ("exiftool")), ("exiftool_merge_keywords", ("exiftool", "sidecar")), ("exiftool_merge_persons", ("exiftool", "sidecar")), @@ -1905,51 +1915,52 @@ def export( export_results = export_photo( photo=p, dest=dest, - verbose=verbose, - export_by_date=export_by_date, - sidecar=sidecar, - sidecar_drop_ext=sidecar_drop_ext, - update=update, - ignore_signature=ignore_signature, - export_as_hardlink=export_as_hardlink, - overwrite=overwrite, - export_edited=export_edited, - skip_original_if_edited=skip_original_if_edited, - original_name=original_name, - export_live=export_live, + album_keyword=album_keyword, + convert_to_jpeg=convert_to_jpeg, + description_template=description_template, + directory=directory, download_missing=download_missing, - exiftool=exiftool, + dry_run=dry_run, + edited_suffix=edited_suffix, exiftool_merge_keywords=exiftool_merge_keywords, exiftool_merge_persons=exiftool_merge_persons, - directory=directory, - filename_template=filename_template, - export_raw=export_raw, - album_keyword=album_keyword, - person_keyword=person_keyword, - keyword_template=keyword_template, - description_template=description_template, - export_db=export_db, - fileutil=fileutil, - dry_run=dry_run, - touch_file=touch_file, - edited_suffix=edited_suffix, - original_suffix=original_suffix, - use_photos_export=use_photos_export, - convert_to_jpeg=convert_to_jpeg, - jpeg_quality=jpeg_quality, - ignore_date_modified=ignore_date_modified, - use_photokit=use_photokit, exiftool_option=exiftool_option, - strip=strip, + exiftool=exiftool, + export_as_hardlink=export_as_hardlink, + export_by_date=export_by_date, + export_db=export_db, + export_dir=dest, + export_edited=export_edited, + export_live=export_live, + export_preview=preview, + export_raw=export_raw, + filename_template=filename_template, + fileutil=fileutil, + force_update=force_update, + ignore_date_modified=ignore_date_modified, + ignore_signature=ignore_signature, jpeg_ext=jpeg_ext, + jpeg_quality=jpeg_quality, + keyword_template=keyword_template, + num_photos=num_photos, + original_name=original_name, + original_suffix=original_suffix, + overwrite=overwrite, + person_keyword=person_keyword, + photo_num=photo_num, + preview_if_missing=preview_if_missing, + preview_suffix=preview_suffix, replace_keywords=replace_keywords, retry=retry, - export_dir=dest, - export_preview=preview, - preview_suffix=preview_suffix, - preview_if_missing=preview_if_missing, - photo_num=photo_num, - num_photos=num_photos, + sidecar_drop_ext=sidecar_drop_ext, + sidecar=sidecar, + skip_original_if_edited=skip_original_if_edited, + strip=strip, + touch_file=touch_file, + update=update, + use_photokit=use_photokit, + use_photos_export=use_photos_export, + verbose=verbose, ) if post_function: @@ -2062,7 +2073,7 @@ def export( fp.close() photo_str_total = "photos" if len(photos) != 1 else "photo" - if update: + if update or force_update: summary = ( f"Processed: {len(photos)} {photo_str_total}, " f"exported: {len(results.new)}, " @@ -2592,6 +2603,7 @@ def export_photo( sidecar=None, sidecar_drop_ext=False, update=None, + force_update=None, ignore_signature=None, export_as_hardlink=None, overwrite=None, @@ -2638,47 +2650,47 @@ def export_photo( Args: photo: PhotoInfo object dest: destination path as string - verbose: boolean; print verbose output - export_by_date: boolean; create export folder in form dest/YYYY/MM/DD - sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export - sidecar_drop_ext: boolean; if True, drops photo extension from sidecar name - export_as_hardlink: boolean; hardlink files instead of copying them - overwrite: boolean; overwrite dest file if it already exists - original_name: boolean; use original filename instead of current filename - export_live: boolean; also export live video component if photo is a live photo - live video will have same name as photo but with .mov extension - download_missing: attempt download of missing iCloud photos - exiftool: use exiftool to write EXIF metadata directly to exported photo + album_keyword: bool; if True, exports album names as keywords in metadata + convert_to_jpeg: bool; if True, converts non-jpeg images to jpeg + description_template: str; optional template string that will be rendered for use as photo description directory: template used to determine output directory - filename_template: template use to determine output file - export_raw: boolean; if True exports raw image associate with the photo - export_edited: boolean; if True exports edited version of photo if there is one - skip_original_if_edited: boolean; if True does not export original if photo has been edited - album_keyword: boolean; if True, exports album names as keywords in metadata - person_keyword: boolean; if True, exports person names as keywords in metadata - keyword_template: list of strings; if provided use rendered template strings as keywords - description_template: string; optional template string that will be rendered for use as photo description - export_db: export database instance compatible with ExportDB_ABC - fileutil: file util class compatible with FileUtilABC - dry_run: boolean; if True, doesn't actually export or update any files - touch_file: boolean; sets file's modification time to match photo date - use_photos_export: boolean; if True forces the use of AppleScript to export even if photo not missing - convert_to_jpeg: boolean; if True, converts non-jpeg images to jpeg - jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0. A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression. - ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set + download_missing: attempt download of missing iCloud photos + dry_run: bool; if True, doesn't actually export or update any files + exiftool_merge_keywords: bool; if True, merged keywords found in file's exif data (requires exiftool) + exiftool_merge_persons: bool; if True, merged persons found in file's exif data (requires exiftool) exiftool_option: optional list flags (e.g. ["-m", "-F"]) to pass to exiftool - exiftool_merge_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool) - exiftool_merge_persons: boolean; if True, merged persons found in file's exif data (requires exiftool) + exiftool: bool; use exiftool to write EXIF metadata directly to exported photo + export_as_hardlink: bool; hardlink files instead of copying them + export_by_date: bool; create export folder in form dest/YYYY/MM/DD + export_db: export database instance compatible with ExportDB_ABC + export_dir: top-level export directory for {export_dir} template + export_edited: bool; if True exports edited version of photo if there is one + export_live: bool; also export live video component if photo is a live photo; live video will have same name as photo but with .mov extension + export_preview: export the preview image generated by Photos + export_raw: bool; if True exports raw image associate with the photo + filename_template: template use to determine output file + fileutil: file util class compatible with FileUtilABC + force_update: bool, only export updated photos but trigger export even if only metadata has changed + ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set jpeg_ext: if not None, specify the extension to use for all JPEG images on export + jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0. A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression. + keyword_template: list of strings; if provided use rendered template strings as keywords + num_photos: int, total number of photos that will be exported + original_name: bool; use original filename instead of current filename + overwrite: bool; overwrite dest file if it already exists + person_keyword: bool; if True, exports person names as keywords in metadata + photo_num: int, which number photo in total of num_photos is being exported + preview_if_missing: bool, export preview if original is missing + preview_suffix: str, template to use as suffix for preview images replace_keywords: if True, --keyword-template replaces keywords instead of adding keywords retry: retry up to retry # of times if there's an error - export_dir: top-level export directory for {export_dir} template - export_preview: export the preview image generated by Photos - preview_suffix: str, template to use as suffix for preview images - preview_if_missing: bool, export preview if original is missing - photo_num: int, which number photo in total of num_photos is being exported - num_photos: int, total number of photos that will be exported - + sidecar_drop_ext: bool; if True, drops photo extension from sidecar name + sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export + skip_original_if_edited: bool; if True does not export original if photo has been edited + touch_file: bool; sets file's modification time to match photo date + update: bool, only export updated photos + use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing + verbose: bool; print verbose output Returns: list of path(s) of exported photo or None if photo was missing @@ -2824,6 +2836,7 @@ def export_photo( export_raw=export_raw, filename=original_filename, fileutil=fileutil, + force_update=force_update, ignore_date_modified=ignore_date_modified, ignore_signature=ignore_signature, jpeg_ext=jpeg_ext, @@ -2936,6 +2949,7 @@ def export_photo( export_raw=not export_original and export_raw, filename=edited_filename, fileutil=fileutil, + force_update=force_update, ignore_date_modified=ignore_date_modified, ignore_signature=ignore_signature, jpeg_ext=jpeg_ext, @@ -3019,6 +3033,7 @@ def export_photo_to_directory( export_raw, filename, fileutil, + force_update, ignore_date_modified, ignore_signature, jpeg_ext, @@ -3077,6 +3092,7 @@ def export_photo_to_directory( export_as_hardlink=export_as_hardlink, export_db=export_db, fileutil=fileutil, + force_update=force_update, ignore_date_modified=ignore_date_modified, ignore_signature=ignore_signature, jpeg_ext=jpeg_ext, @@ -3179,7 +3195,7 @@ def get_filenames_from_template( Args: photo: a PhotoInfo instance filename_template: a PhotoTemplate template string, may be None - original_name: boolean; if True, use photo's original filename instead of current filename + original_name: bool; if True, use photo's original filename instead of current filename dest_path: the path the photo will be exported to strip: if True, strips leading/trailing white space from resulting template edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version @@ -3240,9 +3256,9 @@ def get_dirnames_from_template( Args: photo: a PhotoInstance object directory: a PhotoTemplate template string, may be None - export_by_date: boolean; if True, creates output directories in form YYYY-MM-DD + export_by_date: bool; if True, creates output directories in form YYYY-MM-DD dest: top-level destination directory - dry_run: boolean; if True, runs in dry-run mode and does not create output directories + dry_run: bool; if True, runs in dry-run mode and does not create output directories strip: if True, strips leading/trailing white space from resulting template edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version diff --git a/osxphotos/export_db.py b/osxphotos/export_db.py index 3e5860a6..cf762d72 100644 --- a/osxphotos/export_db.py +++ b/osxphotos/export_db.py @@ -18,8 +18,9 @@ from .utils import normalize_fs_path __all__ = ["ExportDB_ABC", "ExportDBNoOp", "ExportDB", "ExportDBInMemory"] -OSXPHOTOS_EXPORTDB_VERSION = "4.3" +OSXPHOTOS_EXPORTDB_VERSION = "5.0" OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH = "4.3" +OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES = "4.3" OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}" @@ -103,6 +104,14 @@ class ExportDB_ABC(ABC): def set_detected_text_for_uuid(self, uuid, json_text): pass + @abstractmethod + def set_metadata_for_file(self, filename, metadata): + pass + + @abstractmethod + def get_metadata_for_file(self, filename): + pass + @abstractmethod def set_data( self, @@ -114,6 +123,7 @@ class ExportDB_ABC(ABC): edited_stat=None, info_json=None, exif_json=None, + metadata=None, ): pass @@ -183,6 +193,12 @@ class ExportDBNoOp(ExportDB_ABC): def set_detected_text_for_uuid(self, uuid, json_text): pass + def set_metadata_for_file(self, filename, metadata): + pass + + def get_metadata_for_file(self, filename): + pass + def set_data( self, filename, @@ -193,6 +209,7 @@ class ExportDBNoOp(ExportDB_ABC): edited_stat=None, info_json=None, exif_json=None, + metadata=None, ): pass @@ -507,6 +524,39 @@ class ExportDB(ExportDB_ABC): except Error as e: logging.warning(e) + def set_metadata_for_file(self, filename, metadata): + """set metadata of filename in the database""" + filename = str(pathlib.Path(filename).relative_to(self._path)) + filename_normalized = self._normalize_filepath(filename) + conn = self._conn + try: + c = conn.cursor() + c.execute( + "UPDATE files SET metadata = ? WHERE filepath_normalized = ?;", + (metadata, filename_normalized), + ) + conn.commit() + except Error as e: + logging.warning(e) + + def get_metadata_for_file(self, filename): + """get metadata value for file""" + filename = self._normalize_filepath_relative(filename) + conn = self._conn + try: + c = conn.cursor() + c.execute( + "SELECT metadata FROM files WHERE filepath_normalized = ?", + (filename,), + ) + results = c.fetchone() + metadata = results[0] if results else None + except Error as e: + logging.warning(e) + metadata = None + + return metadata + def set_data( self, filename, @@ -517,6 +567,7 @@ class ExportDB(ExportDB_ABC): edited_stat=None, info_json=None, exif_json=None, + metadata=None, ): """sets all the data for file and uuid at once; if any value is None, does not set it""" filename = str(pathlib.Path(filename).relative_to(self._path)) @@ -570,6 +621,15 @@ class ExportDB(ExportDB_ABC): "INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);", (filename_normalized, exif_json), ) + + if metadata is not None: + c.execute( + "UPDATE files " + + "SET metadata = ? " + + "WHERE filepath_normalized = ?;", + (metadata, filename_normalized), + ) + conn.commit() except Error as e: logging.warning(e) @@ -622,7 +682,7 @@ class ExportDB(ExportDB_ABC): conn = self._get_db_connection(dbfile) if not conn: raise Exception("Error getting connection to database {dbfile}") - self._create_db_tables(conn) + self._create_or_migrate_db_tables(conn) self.was_created = True self.was_upgraded = () else: @@ -630,9 +690,7 @@ class ExportDB(ExportDB_ABC): self.was_created = False version_info = self._get_database_version(conn) if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION: - self._create_db_tables(conn) - if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH: - self._migrate_normalized_filepath(conn) + self._create_or_migrate_db_tables(conn) self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION) else: self.was_upgraded = () @@ -664,104 +722,97 @@ class ExportDB(ExportDB_ABC): ).fetchone() return (version_info[0], version_info[1]) - def _create_db_tables(self, conn): - """create (if not already created) the necessary db tables for the export database - conn: sqlite3 db connection + def _create_or_migrate_db_tables(self, conn): + """create (if not already created) the necessary db tables for the export database and apply any needed migrations + + Args: + conn: sqlite3 db connection """ - sql_commands = { - "sql_version_table": """ CREATE TABLE IF NOT EXISTS version ( - id INTEGER PRIMARY KEY, - osxphotos TEXT, - exportdb TEXT - ); """, - "sql_about_table": """ CREATE TABLE IF NOT EXISTS about ( - id INTEGER PRIMARY KEY, - about TEXT - );""", - "sql_files_table": """ CREATE TABLE IF NOT EXISTS files ( - id INTEGER PRIMARY KEY, - filepath TEXT NOT NULL, - filepath_normalized TEXT NOT NULL, - uuid TEXT, - orig_mode INTEGER, - orig_size INTEGER, - orig_mtime REAL, - exif_mode INTEGER, - exif_size INTEGER, - exif_mtime REAL - ); """, - "sql_files_table_migrate": """ CREATE TABLE IF NOT EXISTS files_migrate ( - id INTEGER PRIMARY KEY, - filepath TEXT NOT NULL, - filepath_normalized TEXT NOT NULL, - uuid TEXT, - orig_mode INTEGER, - orig_size INTEGER, - orig_mtime REAL, - exif_mode INTEGER, - exif_size INTEGER, - exif_mtime REAL, - UNIQUE(filepath_normalized) - ); """, - "sql_files_migrate": """ INSERT INTO files_migrate SELECT * FROM files;""", - "sql_files_drop_tables": """ DROP TABLE files;""", - "sql_files_alter": """ ALTER TABLE files_migrate RENAME TO files;""", - "sql_runs_table": """ CREATE TABLE IF NOT EXISTS runs ( - id INTEGER PRIMARY KEY, - datetime TEXT, - python_path TEXT, - script_name TEXT, - args TEXT, - cwd TEXT - ); """, - "sql_info_table": """ CREATE TABLE IF NOT EXISTS info ( - id INTEGER PRIMARY KEY, - uuid text NOT NULL, - json_info JSON - ); """, - "sql_exifdata_table": """ CREATE TABLE IF NOT EXISTS exifdata ( - id INTEGER PRIMARY KEY, - filepath_normalized TEXT NOT NULL, - json_exifdata JSON - ); """, - "sql_edited_table": """ CREATE TABLE IF NOT EXISTS edited ( - id INTEGER PRIMARY KEY, - filepath_normalized TEXT NOT NULL, - mode INTEGER, - size INTEGER, - mtime REAL - ); """, - "sql_converted_table": """ CREATE TABLE IF NOT EXISTS converted ( - id INTEGER PRIMARY KEY, - filepath_normalized TEXT NOT NULL, - mode INTEGER, - size INTEGER, - mtime REAL - ); """, - "sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar ( - id INTEGER PRIMARY KEY, - filepath_normalized TEXT NOT NULL, - sidecar_data TEXT, - mode INTEGER, - size INTEGER, - mtime REAL - ); """, - "sql_detected_text_table": """ CREATE TABLE IF NOT EXISTS detected_text ( - id INTEGER PRIMARY KEY, - uuid TEXT NOT NULL, - text_data JSON - ); """, - "sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """, - "sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """, - "sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """, - "sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""", - "sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""", - "sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""", - "sql_detected_text_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""", - } + try: + version = self._get_database_version(conn) + except Exception as e: + version = (__version__, OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES) + + # Current for version 4.3, for anything greater, do a migration after creation + sql_commands = [ + """ CREATE TABLE IF NOT EXISTS version ( + id INTEGER PRIMARY KEY, + osxphotos TEXT, + exportdb TEXT + ); """, + """ CREATE TABLE IF NOT EXISTS about ( + id INTEGER PRIMARY KEY, + about TEXT + );""", + """ CREATE TABLE IF NOT EXISTS files ( + id INTEGER PRIMARY KEY, + filepath TEXT NOT NULL, + filepath_normalized TEXT NOT NULL, + uuid TEXT, + orig_mode INTEGER, + orig_size INTEGER, + orig_mtime REAL, + exif_mode INTEGER, + exif_size INTEGER, + exif_mtime REAL + ); """, + """ CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY, + datetime TEXT, + python_path TEXT, + script_name TEXT, + args TEXT, + cwd TEXT + ); """, + """ CREATE TABLE IF NOT EXISTS info ( + id INTEGER PRIMARY KEY, + uuid text NOT NULL, + json_info JSON + ); """, + """ CREATE TABLE IF NOT EXISTS exifdata ( + id INTEGER PRIMARY KEY, + filepath_normalized TEXT NOT NULL, + json_exifdata JSON + ); """, + """ CREATE TABLE IF NOT EXISTS edited ( + id INTEGER PRIMARY KEY, + filepath_normalized TEXT NOT NULL, + mode INTEGER, + size INTEGER, + mtime REAL + ); """, + """ CREATE TABLE IF NOT EXISTS converted ( + id INTEGER PRIMARY KEY, + filepath_normalized TEXT NOT NULL, + mode INTEGER, + size INTEGER, + mtime REAL + ); """, + """ CREATE TABLE IF NOT EXISTS sidecar ( + id INTEGER PRIMARY KEY, + filepath_normalized TEXT NOT NULL, + sidecar_data TEXT, + mode INTEGER, + size INTEGER, + mtime REAL + ); """, + """ CREATE TABLE IF NOT EXISTS detected_text ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + text_data JSON + ); """, + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """, + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """, + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """, + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""", + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""", + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""", + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""", + ] + # create the tables if needed try: c = conn.cursor() - for cmd in sql_commands.values(): + for cmd in sql_commands: c.execute(cmd) c.execute( "INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);", @@ -772,6 +823,19 @@ class ExportDB(ExportDB_ABC): except Error as e: logging.warning(e) + # perform needed migrations + if version[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH: + self._migrate_normalized_filepath(conn) + + if version[1] < OSXPHOTOS_EXPORTDB_VERSION: + try: + c = conn.cursor() + # add metadata column to files to support --force-update + c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;") + conn.commit() + except Error as e: + logging.warning(e) + def __del__(self): """ensure the database connection is closed""" try: @@ -810,6 +874,28 @@ class ExportDB(ExportDB_ABC): """Fix all filepath_normalized columns for unicode normalization""" # Prior to database version 4.3, filepath_normalized was not normalized for unicode c = conn.cursor() + migration_sql = [ + """ CREATE TABLE IF NOT EXISTS files_migrate ( + id INTEGER PRIMARY KEY, + filepath TEXT NOT NULL, + filepath_normalized TEXT NOT NULL, + uuid TEXT, + orig_mode INTEGER, + orig_size INTEGER, + orig_mtime REAL, + exif_mode INTEGER, + exif_size INTEGER, + exif_mtime REAL, + UNIQUE(filepath_normalized) + ); """, + """ INSERT INTO files_migrate SELECT * FROM files;""", + """ DROP TABLE files;""", + """ ALTER TABLE files_migrate RENAME TO files;""", + ] + for sql in migration_sql: + c.execute(sql) + conn.commit() + for table in ["converted", "edited", "exifdata", "files", "sidecar"]: old_values = c.execute( f"SELECT filepath_normalized, id FROM {table}" @@ -848,7 +934,7 @@ class ExportDBInMemory(ExportDB): conn = self._get_db_connection() if not conn: raise Exception("Error getting connection to in-memory database") - self._create_db_tables(conn) + self._create_or_migrate_db_tables(conn) self.was_created = True self.was_upgraded = () else: @@ -871,7 +957,7 @@ class ExportDBInMemory(ExportDB): self.was_created = False _, exportdb_ver = self._get_database_version(conn) if exportdb_ver < OSXPHOTOS_EXPORTDB_VERSION: - self._create_db_tables(conn) + self._create_or_migrate_db_tables(conn) self.was_upgraded = (exportdb_ver, OSXPHOTOS_EXPORTDB_VERSION) else: self.was_upgraded = () diff --git a/osxphotos/photoexporter.py b/osxphotos/photoexporter.py index 0f47b37d..264239b6 100644 --- a/osxphotos/photoexporter.py +++ b/osxphotos/photoexporter.py @@ -83,6 +83,7 @@ class ExportOptions: export_as_hardlink: (bool, default=False): if True, will hardlink files instead of copying them export_db: (ExportDB_ABC): instance of a class that conforms to ExportDB_ABC with methods for getting/setting data related to exported files to compare update state fileutil: (FileUtilABC): class that conforms to FileUtilABC with various file utilities + force_update: (bool, default=False): if True, will export photo if any metadata has changed but export otherwise would not be triggered (e.g. metadata changed but not using exiftool) ignore_date_modified (bool): for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set ignore_signature (bool, default=False): ignore file signature when used with update (look only at filename) increment (bool, default=True): if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists @@ -128,6 +129,7 @@ class ExportOptions: export_as_hardlink: bool = False export_db: Optional[ExportDB_ABC] = None fileutil: Optional[FileUtil] = None + force_update: bool = False ignore_date_modified: bool = False ignore_signature: bool = False increment: bool = True @@ -523,7 +525,7 @@ class PhotoExporter: # the export directory preview_name = ( preview_name - if options.overwrite or options.update + if any([options.overwrite, options.update, options.force_update]) else pathlib.Path(increment_filename(preview_name)) ) all_results += self._export_photo( @@ -589,7 +591,7 @@ class PhotoExporter: # if overwrite==False and #increment==False, export should fail if file exists if dest.exists() and not any( - [options.increment, options.update, options.overwrite] + [options.increment, options.update, options.force_update, options.overwrite] ): raise FileExistsError( f"destination exists ({dest}); overwrite={options.overwrite}, increment={options.increment}" @@ -601,11 +603,13 @@ class PhotoExporter: # e.g. exporting sidecar for file1.png and file1.jpeg # if file1.png exists and exporting file1.jpeg, # dest will be file1 (1).jpeg even though file1.jpeg doesn't exist to prevent sidecar collision - if options.increment and not options.update and not options.overwrite: + if options.increment and not any( + [options.update, options.force_update, options.overwrite] + ): return pathlib.Path(increment_filename(dest)) # if update and file exists, need to check to see if it's the write file by checking export db - if options.update and dest.exists() and src: + if (options.update or options.force_update) and dest.exists() and src: export_db = options.export_db fileutil = options.fileutil # destination exists, check to see if destination is the right UUID @@ -735,7 +739,7 @@ class PhotoExporter: # export live_photo .mov file? live_photo = bool(options.live_photo and self.photo.live_photo) - overwrite = options.overwrite or options.update + overwrite = any([options.overwrite, options.update, options.force_update]) # figure out which photo version to request if options.edited or self.photo.shared: @@ -843,7 +847,7 @@ class PhotoExporter: # export live_photo .mov file? live_photo = bool(options.live_photo and self.photo.live_photo) - overwrite = options.overwrite or options.update + overwrite = any([options.overwrite, options.update, options.force_update]) edited_version = options.edited or self.photo.shared # shared photos (in shared albums) show up as not having adjustments (not edited) # but Photos is unable to export the "original" as only a jpeg copy is shared in iCloud @@ -995,16 +999,11 @@ class PhotoExporter: fileutil = options.fileutil export_db = options.export_db - if options.update: # updating + if options.update or options.force_update: # updating cmp_touch, cmp_orig = False, False if dest_exists: # update, destination exists, but we might not need to replace it... - if options.ignore_signature: - cmp_orig = True - cmp_touch = fileutil.cmp( - src, dest, mtime1=int(self.photo.date.timestamp()) - ) - elif options.exiftool: + if options.exiftool: sig_exif = export_db.get_stat_exif_for_file(dest_str) cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif) if cmp_orig: @@ -1026,10 +1025,17 @@ class PhotoExporter: ) cmp_touch = fileutil.cmp_file_sig(dest_str, sig_converted) else: - cmp_orig = fileutil.cmp(src, dest) + cmp_orig = options.ignore_signature or fileutil.cmp(src, dest) cmp_touch = fileutil.cmp( src, dest, mtime1=int(self.photo.date.timestamp()) ) + if options.force_update: + # need to also check the photo's metadata to that in the database + # and if anything changed, we need to update the file + # ony the hex digest of the metadata is stored in the database + cmp_orig = hexdigest( + self.photo.json() + ) == export_db.get_metadata_for_file(dest_str) sig_cmp = cmp_touch if options.touch_file else cmp_orig @@ -1043,7 +1049,7 @@ class PhotoExporter: if sig_edited != (None, None, None) else False ) - sig_cmp = sig_cmp and cmp_edited + sig_cmp = sig_cmp and (options.force_update or cmp_edited) if (options.export_as_hardlink and dest.samefile(src)) or ( not options.export_as_hardlink @@ -1086,7 +1092,9 @@ class PhotoExporter: edited_stat = ( fileutil.file_sig(src) if options.edited else (None, None, None) ) - if dest_exists and (options.update or options.overwrite): + if dest_exists and any( + [options.overwrite, options.update, options.force_update] + ): # need to remove the destination first try: fileutil.unlink(dest) @@ -1129,13 +1137,15 @@ class PhotoExporter: f"Error copying file {src} to {dest_str}: {e} ({lineno(__file__)})" ) from e + json_info = self.photo.json() export_db.set_data( filename=dest_str, uuid=self.photo.uuid, orig_stat=fileutil.file_sig(dest_str), converted_stat=converted_stat, edited_stat=edited_stat, - info_json=self.photo.json(), + info_json=json_info, + metadata=hexdigest(json_info), ) return ExportResults( @@ -1235,10 +1245,13 @@ class PhotoExporter: sidecar_filename ) write_sidecar = ( - not options.update - or (options.update and not sidecar_filename.exists()) + not (options.update or options.force_update) or ( - options.update + (options.update or options.force_update) + and not sidecar_filename.exists() + ) + or ( + (options.update or options.force_update) and (sidecar_digest != old_sidecar_digest) or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig) ) @@ -1333,8 +1346,8 @@ class PhotoExporter: def _should_run_exiftool(self, dest, options: ExportOptions) -> bool: """Return True if exiftool should be run to update metadata""" - run_exiftool = not options.update - if options.update: + run_exiftool = not (options.update or options.force_update) + if options.update or options.force_update: files_are_different = False old_data = options.export_db.get_exifdata_for_file(dest) if old_data is not None: diff --git a/osxphotos/photoinfo.py b/osxphotos/photoinfo.py index cc077d12..17f9500f 100644 --- a/osxphotos/photoinfo.py +++ b/osxphotos/photoinfo.py @@ -1728,7 +1728,8 @@ class PhotoInfo: if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() - return json.dumps(self.asdict(), sort_keys=True, default=default) + dict_data = self.asdict() + return json.dumps(dict_data, sort_keys=True, default=default) def __eq__(self, other): """Compare two PhotoInfo objects for equality""" diff --git a/tests/conftest.py b/tests/conftest.py index 8447d264..20497be8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,7 @@ else: @pytest.fixture(autouse=True) def reset_singletons(): - """ Need to clean up any ExifTool singletons between tests """ + """Need to clean up any ExifTool singletons between tests""" _ExifToolProc.instance = None @@ -73,7 +73,7 @@ def pytest_collection_modifyitems(config, items): def copy_photos_library(photos_library=TEST_LIBRARY, delay=0): - """ copy the test library and open Photos, returns path to copied library """ + """copy the test library and open Photos, returns path to copied library""" script = AppleScript( """ tell application "Photos" @@ -118,3 +118,9 @@ def copy_photos_library(photos_library=TEST_LIBRARY, delay=0): @pytest.fixture def addalbum_library(): copy_photos_library(delay=10) + + +def copy_photos_library_to_path(photos_library_path: str, dest_path: str) -> str: + """Copy a photos library to a folder""" + ditto(photos_library_path, dest_path) + return dest_path diff --git a/tests/test_cli.py b/tests/test_cli.py index 99f4be7c..6efdbe2f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,10 +1,12 @@ -r""" Test the command line interface (CLI) """ +""" Test the command line interface (CLI) """ import os +import sqlite3 import tempfile import pytest from click.testing import CliRunner +from conftest import copy_photos_library_to_path import osxphotos from osxphotos.exiftool import get_exiftool_path @@ -2034,7 +2036,6 @@ def test_export_exiftool_template_change(): assert result.exit_code == 0 assert "updated EXIF data: 1" in result.output - # export with --update, nothing should export result = runner.invoke( export, @@ -2054,6 +2055,7 @@ def test_export_exiftool_template_change(): assert "exported: 0" in result.output assert "updated EXIF data: 0" in result.output + @pytest.mark.skipif(exiftool is None, reason="exiftool not installed") def test_export_exiftool_path(): """test --exiftool with --exiftool-path""" @@ -4809,6 +4811,75 @@ def test_export_update_basic(): ) +def test_export_force_update(): + """test export with --force-update""" + import glob + import os + import os.path + + import osxphotos + from osxphotos.cli import OSXPHOTOS_EXPORT_DB, export + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + # basic export + result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V"]) + assert result.exit_code == 0 + files = glob.glob("*") + assert sorted(files) == sorted(CLI_EXPORT_FILENAMES) + assert os.path.isfile(OSXPHOTOS_EXPORT_DB) + + src = os.path.join(cwd, CLI_PHOTOS_DB) + dest = os.path.join(os.getcwd(), "export_force_update.photoslibrary") + photos_db_path = copy_photos_library_to_path(src, dest) + + # update + result = runner.invoke( + export, [os.path.join(cwd, photos_db_path), ".", "--update"] + ) + assert result.exit_code == 0 + assert ( + f"Processed: {PHOTOS_NOT_IN_TRASH_LEN_15_7} photos, exported: 0, updated: 0, skipped: {PHOTOS_NOT_IN_TRASH_LEN_15_7+PHOTOS_EDITED_15_7}, updated EXIF data: 0, missing: 3, error: 0" + in result.output + ) + + # force update + result = runner.invoke( + export, [os.path.join(cwd, photos_db_path), ".", "--force-update"] + ) + assert result.exit_code == 0 + assert ( + f"Processed: {PHOTOS_NOT_IN_TRASH_LEN_15_7} photos, exported: 0, updated: 0, skipped: {PHOTOS_NOT_IN_TRASH_LEN_15_7+PHOTOS_EDITED_15_7}, updated EXIF data: 0, missing: 3, error: 0" + in result.output + ) + + # update a file + dbpath = os.path.join(photos_db_path, "database/Photos.sqlite") + try: + conn = sqlite3.connect(dbpath) + c = conn.cursor() + except sqlite3.Error as e: + pytest.exit(f"An error occurred opening sqlite file") + + # photo is IMG_4547.jpg + c.execute( + "UPDATE ZADDITIONALASSETATTRIBUTES SET Z_OPT=9, ZTITLE='My Updated Title' WHERE Z_PK=8;" + ) + conn.commit() + + # run again to see if updated metadata forced update + result = runner.invoke( + export, [os.path.join(cwd, photos_db_path), ".", "--force-update"] + ) + assert result.exit_code == 0 + assert ( + f"Processed: {PHOTOS_NOT_IN_TRASH_LEN_15_7} photos, exported: 0, updated: 1, skipped: {PHOTOS_NOT_IN_TRASH_LEN_15_7+PHOTOS_EDITED_15_7-1}, updated EXIF data: 0, missing: 3, error: 0" + in result.output + ) + + @pytest.mark.skipif( "OSXPHOTOS_TEST_EXPORT" not in os.environ, reason="Skip if not running on author's personal library.", @@ -4931,7 +5002,7 @@ def test_export_update_exiftool(): ) assert result.exit_code == 0 assert ( - f"Processed: {PHOTOS_NOT_IN_TRASH_LEN_15_7} photos, exported: 0, updated: 0, skipped: {PHOTOS_NOT_IN_TRASH_LEN_15_7+PHOTOS_EDITED_15_7}, updated EXIF data: 0, missing: 2, error: 0" + f"Processed: {PHOTOS_NOT_IN_TRASH_LEN_15_7} photos, exported: 0, updated: 0, skipped: {PHOTOS_NOT_IN_TRASH_LEN_15_7+PHOTOS_EDITED_15_7}, updated EXIF data: 0, missing: 3, error: 0" in result.output ) diff --git a/tests/test_export_db.py b/tests/test_export_db.py index ebcc6383..75b46f42 100644 --- a/tests/test_export_db.py +++ b/tests/test_export_db.py @@ -7,6 +7,7 @@ import pytest EXIF_DATA = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "EXIF:ImageDescription": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Title": "Elder Park", "EXIF:GPSLatitude": "34 deg 55' 8.01\" S", "EXIF:GPSLongitude": "138 deg 35' 48.70\" E", "Composite:GPSPosition": "34 deg 55' 8.01\" S, 138 deg 35' 48.70\" E", "EXIF:GPSLatitudeRef": "South", "EXIF:GPSLongitudeRef": "East", "EXIF:DateTimeOriginal": "2017:06:20 17:18:56", "EXIF:OffsetTimeOriginal": "+09:30", "EXIF:ModifyDate": "2020:05:18 14:42:04"}]""" INFO_DATA = """{"uuid": "3DD2C897-F19E-4CA6-8C22-B027D5A71907", "filename": "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "original_filename": "IMG_4547.jpg", "date": "2017-06-20T17:18:56.518000+09:30", "description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "title": "Elder Park", "keywords": [], "labels": ["Statue", "Art"], "albums": ["AlbumInFolder"], "folders": {"AlbumInFolder": ["Folder1", "SubFolder2"]}, "persons": [], "path": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/originals/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "ismissing": false, "hasadjustments": true, "external_edit": false, "favorite": false, "hidden": false, "latitude": -34.91889167000001, "longitude": 138.59686167, "path_edited": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/resources/renders/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907_1_201_a.jpeg", "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null, "date_modified": "2020-05-18T14:42:04.608664+09:30", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Elder Park, Adelaide, South Australia, Australia, River Torrens", "names": {"field0": [], "country": ["Australia"], "state_province": ["South Australia"], "sub_administrative_area": ["Adelaide"], "city": ["Adelaide", "Adelaide"], "field5": [], "additional_city_info": ["Adelaide CBD", "Tarndanya"], "ocean": [], "area_of_interest": ["Elder Park", ""], "inland_water": ["River Torrens", "River Torrens"], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": ["River Torrens", "River Torrens"]}, "country_code": "AU", "ishome": false, "address_str": "River Torrens, Adelaide SA, Australia", "address": {"street": null, "sub_locality": "Tarndanya", "city": "Adelaide", "sub_administrative_area": "Adelaide", "state_province": "SA", "postal_code": null, "country": "Australia", "iso_country_code": "AU"}}, "exif": {"flash_fired": false, "iso": 320, "metering_mode": 3, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.058823529411764705, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}""" SIDECAR_DATA = """FOO_BAR""" +METADATA_DATA = "FIZZ" EXIF_DATA2 = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "XMP:Title": "St. James's Park", "XMP:TagsList": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "IPTC:Keywords": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "XMP:Subject": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "EXIF:GPSLatitude": "51 deg 30' 12.86\" N", "EXIF:GPSLongitude": "0 deg 7' 54.50\" W", "Composite:GPSPosition": "51 deg 30' 12.86\" N, 0 deg 7' 54.50\" W", "EXIF:GPSLatitudeRef": "North", "EXIF:GPSLongitudeRef": "West", "EXIF:DateTimeOriginal": "2018:10:13 09:18:12", "EXIF:OffsetTimeOriginal": "-04:00", "EXIF:ModifyDate": "2019:12:08 14:06:44"}]""" INFO_DATA2 = """{"uuid": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529", "filename": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "original_filename": "IMG_8440.JPG", "date": "2019-06-11T11:42:06.711805-07:00", "description": null, "title": null, "keywords": [], "labels": ["Sky", "Cloudy", "Fence", "Land", "Outdoor", "Park", "Amusement Park", "Roller Coaster"], "albums": [], "folders": {}, "persons": [], "path": "/Volumes/MacBook Catalina - Data/Users/rhet/Pictures/Photos Library.photoslibrary/originals/F/F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 33.81558666666667, "longitude": -117.99298, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": true, "incloud": true, "date_modified": "2019-10-14T00:51:47.141950-07:00", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Adventure City, Stanton, California, United States", "names": {"field0": [], "country": ["United States"], "state_province": ["California"], "sub_administrative_area": ["Orange"], "city": ["Stanton", "Anaheim", "Anaheim"], "field5": [], "additional_city_info": ["West Anaheim"], "ocean": [], "area_of_interest": ["Adventure City", "Adventure City"], "inland_water": [], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": []}, "country_code": "US", "ishome": false, "address_str": "Adventure City, 1240 S Beach Blvd, Anaheim, CA 92804, United States", "address": {"street": "1240 S Beach Blvd", "sub_locality": "West Anaheim", "city": "Stanton", "sub_administrative_area": "Orange", "state_province": "CA", "postal_code": "92804", "country": "United States", "iso_country_code": "US"}}, "exif": {"flash_fired": false, "iso": 25, "metering_mode": 5, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.0004940711462450593, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}""" @@ -64,6 +65,7 @@ def test_export_db(): (10, 11, 12), INFO_DATA, EXIF_DATA, + METADATA_DATA, ) assert db.get_uuid_for_file(filepath2) == "BAR-FOO" assert db.get_info_for_uuid("BAR-FOO") == INFO_DATA @@ -73,6 +75,7 @@ def test_export_db(): assert db.get_stat_converted_for_file(filepath2) == (7, 8, 9) assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12) assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"]) + assert db.get_metadata_for_file(filepath2) == METADATA_DATA # test set_data value=None doesn't overwrite existing data db.set_data( @@ -84,6 +87,7 @@ def test_export_db(): None, None, None, + None, ) assert db.get_uuid_for_file(filepath2) == "BAR-FOO" assert db.get_info_for_uuid("BAR-FOO") == INFO_DATA @@ -93,6 +97,7 @@ def test_export_db(): assert db.get_stat_converted_for_file(filepath2) == (7, 8, 9) assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12) assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"]) + assert db.get_metadata_for_file(filepath2) == METADATA_DATA # close and re-open db.close() @@ -107,6 +112,8 @@ def test_export_db(): assert db.get_stat_edited_for_file(filepath2) == (10, 11, 12) assert sorted(db.get_previous_uuids()) == (["BAR-FOO", "FOO-BAR"]) assert json.loads(db.get_detected_text_for_uuid("FOO-BAR")) == [["foo", 0.5]] + assert db.get_metadata_for_file(filepath2) == METADATA_DATA + # update data db.set_uuid_for_file(filepath, "FUBAR") @@ -148,9 +155,10 @@ def test_export_db_no_op(): db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15)) assert db.get_sidecar_for_file(filepath) == (None, (None, None, None)) assert db.get_previous_uuids() == [] - db.set_detected_text_for_uuid("FOO-BAR", json.dumps([["foo", 0.5]])) assert db.get_detected_text_for_uuid("FOO-BAR") is None + db.set_metadata_for_file(filepath, METADATA_DATA) + assert db.get_metadata_for_file(filepath) is None # test set_data which sets all at the same time filepath2 = os.path.join(tempdir.name, "test2.jpg") @@ -163,6 +171,7 @@ def test_export_db_no_op(): (10, 11, 12), INFO_DATA, EXIF_DATA, + METADATA_DATA, ) assert db.get_uuid_for_file(filepath2) is None assert db.get_info_for_uuid("BAR-FOO") is None @@ -172,6 +181,7 @@ def test_export_db_no_op(): assert db.get_stat_converted_for_file(filepath) is None assert db.get_stat_edited_for_file(filepath) is None assert db.get_previous_uuids() == [] + assert db.get_metadata_for_file(filepath) is None # update data db.set_uuid_for_file(filepath, "FUBAR") @@ -207,7 +217,7 @@ def test_export_db_in_memory(): db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15)) assert db.get_previous_uuids() == ["FOO-BAR"] db.set_detected_text_for_uuid("FOO-BAR", json.dumps([["foo", 0.5]])) - + db.set_metadata_for_file(filepath, METADATA_DATA) db.close() dbram = ExportDBInMemory(dbname, tempdir.name) @@ -226,6 +236,7 @@ def test_export_db_in_memory(): assert dbram.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15)) assert dbram.get_previous_uuids() == ["FOO-BAR"] assert json.loads(dbram.get_detected_text_for_uuid("FOO-BAR")) == [["foo", 0.5]] + assert dbram.get_metadata_for_file(filepath) == METADATA_DATA # change a value dbram.set_uuid_for_file(filepath, "FUBAR") @@ -237,6 +248,7 @@ def test_export_db_in_memory(): dbram.set_stat_edited_for_file(filepath, (4, 5, 6)) dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) dbram.set_detected_text_for_uuid("FUBAR", json.dumps([["bar", 0.5]])) + dbram.set_metadata_for_file(filepath, "FUBAR") assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR" assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2 @@ -248,6 +260,7 @@ def test_export_db_in_memory(): assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) assert dbram.get_previous_uuids() == ["FUBAR"] assert json.loads(dbram.get_detected_text_for_uuid("FUBAR")) == [["bar", 0.5]] + assert dbram.get_metadata_for_file(filepath) == "FUBAR" dbram.close() @@ -265,6 +278,7 @@ def test_export_db_in_memory(): assert db.get_info_for_uuid("FUBAR") is None assert db.get_detected_text_for_uuid("FUBAR") is None + assert db.get_metadata_for_file(filepath) == METADATA_DATA def test_export_db_in_memory_nofile(): @@ -295,6 +309,7 @@ def test_export_db_in_memory_nofile(): dbram.set_stat_edited_for_file(filepath, (4, 5, 6)) dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22)) dbram.set_detected_text_for_uuid("FUBAR", json.dumps([["bar", 0.5]])) + dbram.set_metadata_for_file(filepath, METADATA_DATA) assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR" assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2 @@ -306,5 +321,6 @@ def test_export_db_in_memory_nofile(): assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22)) assert dbram.get_previous_uuids() == ["FUBAR"] assert json.loads(dbram.get_detected_text_for_uuid("FUBAR")) == [["bar", 0.5]] + assert dbram.get_metadata_for_file(filepath) == METADATA_DATA dbram.close()