forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added --force-update, #621

Browse Source
This commit is contained in:
Rhet Turnbull
2022-02-12 17:49:40 -08:00
parent ac4083bfbb
commit bfa888adc5
17 changed files with 559 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.45.8"
__version__ = "0.45.9"

412
osxphotos/cli.py
View File

@@ -691,7 +691,15 @@ def cli(ctx, db, json_, debug):
@click.option(
"--update",
is_flag=True,
help="Only export new or updated files. See notes below on export and --update.",
help="Only export new or updated files. "
"See also --force-update and notes below on export and --update.",
)
@click.option(
"--force-update",
is_flag=True,
help="Only export new or updated files. Unlike --update, --force-update will re-export photos "
"if their metadata has changed even if this would not otherwise trigger an export. "
"See also --update and notes below on export and --update.",
)
@click.option(
"--ignore-signature",
@@ -1235,6 +1243,7 @@ def export(
timestamp,
missing,
update,
force_update,
ignore_signature,
only_new,
dry_run,
@@ -1398,133 +1407,134 @@ def export(
# re-set the local vars to the corresponding config value
# this isn't elegant but avoids having to rewrite this function to use cfg.varname for every parameter
db = cfg.db
photos_library = cfg.photos_library
keyword = cfg.keyword
person = cfg.person
add_exported_to_album = cfg.add_exported_to_album
add_missing_to_album = cfg.add_missing_to_album
add_skipped_to_album = cfg.add_skipped_to_album
album = cfg.album
folder = cfg.folder
name = cfg.name
uuid = cfg.uuid
uuid_from_file = cfg.uuid_from_file
title = cfg.title
no_title = cfg.no_title
description = cfg.description
no_description = cfg.no_description
uti = cfg.uti
ignore_case = cfg.ignore_case
edited = cfg.edited
external_edit = cfg.external_edit
favorite = cfg.favorite
not_favorite = cfg.not_favorite
hidden = cfg.hidden
not_hidden = cfg.not_hidden
shared = cfg.shared
not_shared = cfg.not_shared
from_date = cfg.from_date
to_date = cfg.to_date
from_time = cfg.from_time
to_time = cfg.to_time
verbose = cfg.verbose
missing = cfg.missing
update = cfg.update
ignore_signature = cfg.ignore_signature
dry_run = cfg.dry_run
export_as_hardlink = cfg.export_as_hardlink
touch_file = cfg.touch_file
overwrite = cfg.overwrite
retry = cfg.retry
export_by_date = cfg.export_by_date
skip_edited = cfg.skip_edited
skip_original_if_edited = cfg.skip_original_if_edited
skip_bursts = cfg.skip_bursts
skip_live = cfg.skip_live
skip_raw = cfg.skip_raw
skip_uuid = cfg.skip_uuid
skip_uuid_from_file = cfg.skip_uuid_from_file
person_keyword = cfg.person_keyword
album_keyword = cfg.album_keyword
keyword_template = cfg.keyword_template
replace_keywords = cfg.replace_keywords
description_template = cfg.description_template
finder_tag_template = cfg.finder_tag_template
finder_tag_keywords = cfg.finder_tag_keywords
xattr_template = cfg.xattr_template
current_name = cfg.current_name
convert_to_jpeg = cfg.convert_to_jpeg
jpeg_quality = cfg.jpeg_quality
sidecar = cfg.sidecar
sidecar_drop_ext = cfg.sidecar_drop_ext
only_photos = cfg.only_photos
only_movies = cfg.only_movies
beta = cfg.beta
burst = cfg.burst
not_burst = cfg.not_burst
live = cfg.live
not_live = cfg.not_live
download_missing = cfg.download_missing
exiftool = cfg.exiftool
exiftool_path = cfg.exiftool_path
exiftool_option = cfg.exiftool_option
exiftool_merge_keywords = cfg.exiftool_merge_keywords
exiftool_merge_persons = cfg.exiftool_merge_persons
ignore_date_modified = cfg.ignore_date_modified
portrait = cfg.portrait
not_portrait = cfg.not_portrait
screenshot = cfg.screenshot
not_screenshot = cfg.not_screenshot
slow_mo = cfg.slow_mo
not_slow_mo = cfg.not_slow_mo
time_lapse = cfg.time_lapse
not_time_lapse = cfg.not_time_lapse
hdr = cfg.hdr
not_hdr = cfg.not_hdr
selfie = cfg.selfie
not_selfie = cfg.not_selfie
panorama = cfg.panorama
not_panorama = cfg.not_panorama
has_raw = cfg.has_raw
directory = cfg.directory
filename_template = cfg.filename_template
jpeg_ext = cfg.jpeg_ext
strip = cfg.strip
edited_suffix = cfg.edited_suffix
original_suffix = cfg.original_suffix
place = cfg.place
no_place = cfg.no_place
location = cfg.location
no_location = cfg.no_location
has_comment = cfg.has_comment
no_comment = cfg.no_comment
has_likes = cfg.has_likes
no_likes = cfg.no_likes
label = cfg.label
cleanup = cfg.cleanup
convert_to_jpeg = cfg.convert_to_jpeg
current_name = cfg.current_name
db = cfg.db
deleted = cfg.deleted
deleted_only = cfg.deleted_only
use_photos_export = cfg.use_photos_export
use_photokit = cfg.use_photokit
report = cfg.report
cleanup = cfg.cleanup
add_exported_to_album = cfg.add_exported_to_album
add_skipped_to_album = cfg.add_skipped_to_album
add_missing_to_album = cfg.add_missing_to_album
exportdb = cfg.exportdb
beta = cfg.beta
only_new = cfg.only_new
in_album = cfg.in_album
not_in_album = cfg.not_in_album
min_size = cfg.min_size
max_size = cfg.max_size
regex = cfg.regex
selected = cfg.selected
exif = cfg.exif
query_eval = cfg.query_eval
query_function = cfg.query_function
description = cfg.description
description_template = cfg.description_template
directory = cfg.directory
download_missing = cfg.download_missing
dry_run = cfg.dry_run
duplicate = cfg.duplicate
edited = cfg.edited
edited_suffix = cfg.edited_suffix
exif = cfg.exif
exiftool = cfg.exiftool
exiftool_merge_keywords = cfg.exiftool_merge_keywords
exiftool_merge_persons = cfg.exiftool_merge_persons
exiftool_option = cfg.exiftool_option
exiftool_path = cfg.exiftool_path
export_as_hardlink = cfg.export_as_hardlink
export_by_date = cfg.export_by_date
exportdb = cfg.exportdb
external_edit = cfg.external_edit
favorite = cfg.favorite
filename_template = cfg.filename_template
finder_tag_keywords = cfg.finder_tag_keywords
finder_tag_template = cfg.finder_tag_template
folder = cfg.folder
force_update = cfg.force_update
from_date = cfg.from_date
from_time = cfg.from_time
has_comment = cfg.has_comment
has_likes = cfg.has_likes
has_raw = cfg.has_raw
hdr = cfg.hdr
hidden = cfg.hidden
ignore_case = cfg.ignore_case
ignore_date_modified = cfg.ignore_date_modified
ignore_signature = cfg.ignore_signature
in_album = cfg.in_album
jpeg_ext = cfg.jpeg_ext
jpeg_quality = cfg.jpeg_quality
keyword = cfg.keyword
keyword_template = cfg.keyword_template
label = cfg.label
live = cfg.live
location = cfg.location
max_size = cfg.max_size
min_size = cfg.min_size
missing = cfg.missing
name = cfg.name
no_comment = cfg.no_comment
no_description = cfg.no_description
no_likes = cfg.no_likes
no_location = cfg.no_location
no_place = cfg.no_place
no_title = cfg.no_title
not_burst = cfg.not_burst
not_favorite = cfg.not_favorite
not_hdr = cfg.not_hdr
not_hidden = cfg.not_hidden
not_in_album = cfg.not_in_album
not_live = cfg.not_live
not_panorama = cfg.not_panorama
not_portrait = cfg.not_portrait
not_screenshot = cfg.not_screenshot
not_selfie = cfg.not_selfie
not_shared = cfg.not_shared
not_slow_mo = cfg.not_slow_mo
not_time_lapse = cfg.not_time_lapse
only_movies = cfg.only_movies
only_new = cfg.only_new
only_photos = cfg.only_photos
original_suffix = cfg.original_suffix
overwrite = cfg.overwrite
panorama = cfg.panorama
person = cfg.person
person_keyword = cfg.person_keyword
photos_library = cfg.photos_library
place = cfg.place
portrait = cfg.portrait
post_command = cfg.post_command
post_function = cfg.post_function
preview = cfg.preview
preview_suffix = cfg.preview_suffix
preview_if_missing = cfg.preview_if_missing
preview_suffix = cfg.preview_suffix
query_eval = cfg.query_eval
query_function = cfg.query_function
regex = cfg.regex
replace_keywords = cfg.replace_keywords
report = cfg.report
retry = cfg.retry
screenshot = cfg.screenshot
selected = cfg.selected
selfie = cfg.selfie
shared = cfg.shared
sidecar = cfg.sidecar
sidecar_drop_ext = cfg.sidecar_drop_ext
skip_bursts = cfg.skip_bursts
skip_edited = cfg.skip_edited
skip_live = cfg.skip_live
skip_original_if_edited = cfg.skip_original_if_edited
skip_raw = cfg.skip_raw
skip_uuid = cfg.skip_uuid
skip_uuid_from_file = cfg.skip_uuid_from_file
slow_mo = cfg.slow_mo
strip = cfg.strip
time_lapse = cfg.time_lapse
title = cfg.title
to_date = cfg.to_date
to_time = cfg.to_time
touch_file = cfg.touch_file
update = cfg.update
use_photokit = cfg.use_photokit
use_photos_export = cfg.use_photos_export
uti = cfg.uti
uuid = cfg.uuid
uuid_from_file = cfg.uuid_from_file
verbose = cfg.verbose
xattr_template = cfg.xattr_template
# config file might have changed verbose
VERBOSE = bool(verbose)
@@ -1564,8 +1574,8 @@ def export(
dependent_options = [
("missing", ("download_missing", "use_photos_export")),
("jpeg_quality", ("convert_to_jpeg")),
("ignore_signature", ("update")),
("only_new", ("update")),
("ignore_signature", ("update", "force_update")),
("only_new", ("update", "force_update")),
("exiftool_option", ("exiftool")),
("exiftool_merge_keywords", ("exiftool", "sidecar")),
("exiftool_merge_persons", ("exiftool", "sidecar")),
@@ -1905,51 +1915,52 @@ def export(
export_results = export_photo(
photo=p,
dest=dest,
verbose=verbose,
export_by_date=export_by_date,
sidecar=sidecar,
sidecar_drop_ext=sidecar_drop_ext,
update=update,
ignore_signature=ignore_signature,
export_as_hardlink=export_as_hardlink,
overwrite=overwrite,
export_edited=export_edited,
skip_original_if_edited=skip_original_if_edited,
original_name=original_name,
export_live=export_live,
album_keyword=album_keyword,
convert_to_jpeg=convert_to_jpeg,
description_template=description_template,
directory=directory,
download_missing=download_missing,
exiftool=exiftool,
dry_run=dry_run,
edited_suffix=edited_suffix,
exiftool_merge_keywords=exiftool_merge_keywords,
exiftool_merge_persons=exiftool_merge_persons,
directory=directory,
filename_template=filename_template,
export_raw=export_raw,
album_keyword=album_keyword,
person_keyword=person_keyword,
keyword_template=keyword_template,
description_template=description_template,
export_db=export_db,
fileutil=fileutil,
dry_run=dry_run,
touch_file=touch_file,
edited_suffix=edited_suffix,
original_suffix=original_suffix,
use_photos_export=use_photos_export,
convert_to_jpeg=convert_to_jpeg,
jpeg_quality=jpeg_quality,
ignore_date_modified=ignore_date_modified,
use_photokit=use_photokit,
exiftool_option=exiftool_option,
strip=strip,
exiftool=exiftool,
export_as_hardlink=export_as_hardlink,
export_by_date=export_by_date,
export_db=export_db,
export_dir=dest,
export_edited=export_edited,
export_live=export_live,
export_preview=preview,
export_raw=export_raw,
filename_template=filename_template,
fileutil=fileutil,
force_update=force_update,
ignore_date_modified=ignore_date_modified,
ignore_signature=ignore_signature,
jpeg_ext=jpeg_ext,
jpeg_quality=jpeg_quality,
keyword_template=keyword_template,
num_photos=num_photos,
original_name=original_name,
original_suffix=original_suffix,
overwrite=overwrite,
person_keyword=person_keyword,
photo_num=photo_num,
preview_if_missing=preview_if_missing,
preview_suffix=preview_suffix,
replace_keywords=replace_keywords,
retry=retry,
export_dir=dest,
export_preview=preview,
preview_suffix=preview_suffix,
preview_if_missing=preview_if_missing,
photo_num=photo_num,
num_photos=num_photos,
sidecar_drop_ext=sidecar_drop_ext,
sidecar=sidecar,
skip_original_if_edited=skip_original_if_edited,
strip=strip,
touch_file=touch_file,
update=update,
use_photokit=use_photokit,
use_photos_export=use_photos_export,
verbose=verbose,
)
if post_function:
@@ -2062,7 +2073,7 @@ def export(
fp.close()
photo_str_total = "photos" if len(photos) != 1 else "photo"
if update:
if update or force_update:
summary = (
f"Processed: {len(photos)} {photo_str_total}, "
f"exported: {len(results.new)}, "
@@ -2592,6 +2603,7 @@ def export_photo(
sidecar=None,
sidecar_drop_ext=False,
update=None,
force_update=None,
ignore_signature=None,
export_as_hardlink=None,
overwrite=None,
@@ -2638,47 +2650,47 @@ def export_photo(
Args:
photo: PhotoInfo object
dest: destination path as string
verbose: boolean; print verbose output
export_by_date: boolean; create export folder in form dest/YYYY/MM/DD
sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export
sidecar_drop_ext: boolean; if True, drops photo extension from sidecar name
export_as_hardlink: boolean; hardlink files instead of copying them
overwrite: boolean; overwrite dest file if it already exists
original_name: boolean; use original filename instead of current filename
export_live: boolean; also export live video component if photo is a live photo
live video will have same name as photo but with .mov extension
download_missing: attempt download of missing iCloud photos
exiftool: use exiftool to write EXIF metadata directly to exported photo
album_keyword: bool; if True, exports album names as keywords in metadata
convert_to_jpeg: bool; if True, converts non-jpeg images to jpeg
description_template: str; optional template string that will be rendered for use as photo description
directory: template used to determine output directory
filename_template: template use to determine output file
export_raw: boolean; if True exports raw image associate with the photo
export_edited: boolean; if True exports edited version of photo if there is one
skip_original_if_edited: boolean; if True does not export original if photo has been edited
album_keyword: boolean; if True, exports album names as keywords in metadata
person_keyword: boolean; if True, exports person names as keywords in metadata
keyword_template: list of strings; if provided use rendered template strings as keywords
description_template: string; optional template string that will be rendered for use as photo description
export_db: export database instance compatible with ExportDB_ABC
fileutil: file util class compatible with FileUtilABC
dry_run: boolean; if True, doesn't actually export or update any files
touch_file: boolean; sets file's modification time to match photo date
use_photos_export: boolean; if True forces the use of AppleScript to export even if photo not missing
convert_to_jpeg: boolean; if True, converts non-jpeg images to jpeg
jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0. A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
download_missing: attempt download of missing iCloud photos
dry_run: bool; if True, doesn't actually export or update any files
exiftool_merge_keywords: bool; if True, merged keywords found in file's exif data (requires exiftool)
exiftool_merge_persons: bool; if True, merged persons found in file's exif data (requires exiftool)
exiftool_option: optional list flags (e.g. ["-m", "-F"]) to pass to exiftool
exiftool_merge_keywords: boolean; if True, merged keywords found in file's exif data (requires exiftool)
exiftool_merge_persons: boolean; if True, merged persons found in file's exif data (requires exiftool)
exiftool: bool; use exiftool to write EXIF metadata directly to exported photo
export_as_hardlink: bool; hardlink files instead of copying them
export_by_date: bool; create export folder in form dest/YYYY/MM/DD
export_db: export database instance compatible with ExportDB_ABC
export_dir: top-level export directory for {export_dir} template
export_edited: bool; if True exports edited version of photo if there is one
export_live: bool; also export live video component if photo is a live photo; live video will have same name as photo but with .mov extension
export_preview: export the preview image generated by Photos
export_raw: bool; if True exports raw image associate with the photo
filename_template: template use to determine output file
fileutil: file util class compatible with FileUtilABC
force_update: bool, only export updated photos but trigger export even if only metadata has changed
ignore_date_modified: if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
jpeg_ext: if not None, specify the extension to use for all JPEG images on export
jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0. A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
keyword_template: list of strings; if provided use rendered template strings as keywords
num_photos: int, total number of photos that will be exported
original_name: bool; use original filename instead of current filename
overwrite: bool; overwrite dest file if it already exists
person_keyword: bool; if True, exports person names as keywords in metadata
photo_num: int, which number photo in total of num_photos is being exported
preview_if_missing: bool, export preview if original is missing
preview_suffix: str, template to use as suffix for preview images
replace_keywords: if True, --keyword-template replaces keywords instead of adding keywords
retry: retry up to retry # of times if there's an error
export_dir: top-level export directory for {export_dir} template
export_preview: export the preview image generated by Photos
preview_suffix: str, template to use as suffix for preview images
preview_if_missing: bool, export preview if original is missing
photo_num: int, which number photo in total of num_photos is being exported
num_photos: int, total number of photos that will be exported
sidecar_drop_ext: bool; if True, drops photo extension from sidecar name
sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export
skip_original_if_edited: bool; if True does not export original if photo has been edited
touch_file: bool; sets file's modification time to match photo date
update: bool, only export updated photos
use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing
verbose: bool; print verbose output
Returns:
list of path(s) of exported photo or None if photo was missing
@@ -2824,6 +2836,7 @@ def export_photo(
export_raw=export_raw,
filename=original_filename,
fileutil=fileutil,
force_update=force_update,
ignore_date_modified=ignore_date_modified,
ignore_signature=ignore_signature,
jpeg_ext=jpeg_ext,
@@ -2936,6 +2949,7 @@ def export_photo(
export_raw=not export_original and export_raw,
filename=edited_filename,
fileutil=fileutil,
force_update=force_update,
ignore_date_modified=ignore_date_modified,
ignore_signature=ignore_signature,
jpeg_ext=jpeg_ext,
@@ -3019,6 +3033,7 @@ def export_photo_to_directory(
export_raw,
filename,
fileutil,
force_update,
ignore_date_modified,
ignore_signature,
jpeg_ext,
@@ -3077,6 +3092,7 @@ def export_photo_to_directory(
export_as_hardlink=export_as_hardlink,
export_db=export_db,
fileutil=fileutil,
force_update=force_update,
ignore_date_modified=ignore_date_modified,
ignore_signature=ignore_signature,
jpeg_ext=jpeg_ext,
@@ -3179,7 +3195,7 @@ def get_filenames_from_template(
Args:
photo: a PhotoInfo instance
filename_template: a PhotoTemplate template string, may be None
original_name: boolean; if True, use photo's original filename instead of current filename
original_name: bool; if True, use photo's original filename instead of current filename
dest_path: the path the photo will be exported to
strip: if True, strips leading/trailing white space from resulting template
edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version
@@ -3240,9 +3256,9 @@ def get_dirnames_from_template(
Args:
photo: a PhotoInstance object
directory: a PhotoTemplate template string, may be None
export_by_date: boolean; if True, creates output directories in form YYYY-MM-DD
export_by_date: bool; if True, creates output directories in form YYYY-MM-DD
dest: top-level destination directory
dry_run: boolean; if True, runs in dry-run mode and does not create output directories
dry_run: bool; if True, runs in dry-run mode and does not create output directories
strip: if True, strips leading/trailing white space from resulting template
edited: if True, sets {edited_version} field to True, otherwise it gets set to False; set if you want template evaluated for edited version

290
osxphotos/export_db.py
View File

@@ -18,8 +18,9 @@ from .utils import normalize_fs_path
__all__ = ["ExportDB_ABC", "ExportDBNoOp", "ExportDB", "ExportDBInMemory"]
OSXPHOTOS_EXPORTDB_VERSION = "4.3"
OSXPHOTOS_EXPORTDB_VERSION = "5.0"
OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH = "4.3"
OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES = "4.3"
OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
@@ -103,6 +104,14 @@ class ExportDB_ABC(ABC):
def set_detected_text_for_uuid(self, uuid, json_text):
pass
@abstractmethod
def set_metadata_for_file(self, filename, metadata):
pass
@abstractmethod
def get_metadata_for_file(self, filename):
pass
@abstractmethod
def set_data(
self,
@@ -114,6 +123,7 @@ class ExportDB_ABC(ABC):
edited_stat=None,
info_json=None,
exif_json=None,
metadata=None,
):
pass
@@ -183,6 +193,12 @@ class ExportDBNoOp(ExportDB_ABC):
def set_detected_text_for_uuid(self, uuid, json_text):
pass
def set_metadata_for_file(self, filename, metadata):
pass
def get_metadata_for_file(self, filename):
pass
def set_data(
self,
filename,
@@ -193,6 +209,7 @@ class ExportDBNoOp(ExportDB_ABC):
edited_stat=None,
info_json=None,
exif_json=None,
metadata=None,
):
pass
@@ -507,6 +524,39 @@ class ExportDB(ExportDB_ABC):
except Error as e:
logging.warning(e)
def set_metadata_for_file(self, filename, metadata):
"""set metadata of filename in the database"""
filename = str(pathlib.Path(filename).relative_to(self._path))
filename_normalized = self._normalize_filepath(filename)
conn = self._conn
try:
c = conn.cursor()
c.execute(
"UPDATE files SET metadata = ? WHERE filepath_normalized = ?;",
(metadata, filename_normalized),
)
conn.commit()
except Error as e:
logging.warning(e)
def get_metadata_for_file(self, filename):
"""get metadata value for file"""
filename = self._normalize_filepath_relative(filename)
conn = self._conn
try:
c = conn.cursor()
c.execute(
"SELECT metadata FROM files WHERE filepath_normalized = ?",
(filename,),
)
results = c.fetchone()
metadata = results[0] if results else None
except Error as e:
logging.warning(e)
metadata = None
return metadata
def set_data(
self,
filename,
@@ -517,6 +567,7 @@ class ExportDB(ExportDB_ABC):
edited_stat=None,
info_json=None,
exif_json=None,
metadata=None,
):
"""sets all the data for file and uuid at once; if any value is None, does not set it"""
filename = str(pathlib.Path(filename).relative_to(self._path))
@@ -570,6 +621,15 @@ class ExportDB(ExportDB_ABC):
"INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
(filename_normalized, exif_json),
)
if metadata is not None:
c.execute(
"UPDATE files "
+ "SET metadata = ? "
+ "WHERE filepath_normalized = ?;",
(metadata, filename_normalized),
)
conn.commit()
except Error as e:
logging.warning(e)
@@ -622,7 +682,7 @@ class ExportDB(ExportDB_ABC):
conn = self._get_db_connection(dbfile)
if not conn:
raise Exception("Error getting connection to database {dbfile}")
self._create_db_tables(conn)
self._create_or_migrate_db_tables(conn)
self.was_created = True
self.was_upgraded = ()
else:
@@ -630,9 +690,7 @@ class ExportDB(ExportDB_ABC):
self.was_created = False
version_info = self._get_database_version(conn)
if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION:
self._create_db_tables(conn)
if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH:
self._migrate_normalized_filepath(conn)
self._create_or_migrate_db_tables(conn)
self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
else:
self.was_upgraded = ()
@@ -664,104 +722,97 @@ class ExportDB(ExportDB_ABC):
).fetchone()
return (version_info[0], version_info[1])
def _create_db_tables(self, conn):
"""create (if not already created) the necessary db tables for the export database
conn: sqlite3 db connection
def _create_or_migrate_db_tables(self, conn):
"""create (if not already created) the necessary db tables for the export database and apply any needed migrations
Args:
conn: sqlite3 db connection
"""
sql_commands = {
"sql_version_table": """ CREATE TABLE IF NOT EXISTS version (
id INTEGER PRIMARY KEY,
osxphotos TEXT,
exportdb TEXT
); """,
"sql_about_table": """ CREATE TABLE IF NOT EXISTS about (
id INTEGER PRIMARY KEY,
about TEXT
);""",
"sql_files_table": """ CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY,
filepath TEXT NOT NULL,
filepath_normalized TEXT NOT NULL,
uuid TEXT,
orig_mode INTEGER,
orig_size INTEGER,
orig_mtime REAL,
exif_mode INTEGER,
exif_size INTEGER,
exif_mtime REAL
); """,
"sql_files_table_migrate": """ CREATE TABLE IF NOT EXISTS files_migrate (
id INTEGER PRIMARY KEY,
filepath TEXT NOT NULL,
filepath_normalized TEXT NOT NULL,
uuid TEXT,
orig_mode INTEGER,
orig_size INTEGER,
orig_mtime REAL,
exif_mode INTEGER,
exif_size INTEGER,
exif_mtime REAL,
UNIQUE(filepath_normalized)
); """,
"sql_files_migrate": """ INSERT INTO files_migrate SELECT * FROM files;""",
"sql_files_drop_tables": """ DROP TABLE files;""",
"sql_files_alter": """ ALTER TABLE files_migrate RENAME TO files;""",
"sql_runs_table": """ CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY,
datetime TEXT,
python_path TEXT,
script_name TEXT,
args TEXT,
cwd TEXT
); """,
"sql_info_table": """ CREATE TABLE IF NOT EXISTS info (
id INTEGER PRIMARY KEY,
uuid text NOT NULL,
json_info JSON
); """,
"sql_exifdata_table": """ CREATE TABLE IF NOT EXISTS exifdata (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
json_exifdata JSON
); """,
"sql_edited_table": """ CREATE TABLE IF NOT EXISTS edited (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
"sql_converted_table": """ CREATE TABLE IF NOT EXISTS converted (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
"sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
sidecar_data TEXT,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
"sql_detected_text_table": """ CREATE TABLE IF NOT EXISTS detected_text (
id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL,
text_data JSON
); """,
"sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
"sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
"sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
"sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
"sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
"sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
"sql_detected_text_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
}
try:
version = self._get_database_version(conn)
except Exception as e:
version = (__version__, OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_TABLES)
# Current for version 4.3, for anything greater, do a migration after creation
sql_commands = [
""" CREATE TABLE IF NOT EXISTS version (
id INTEGER PRIMARY KEY,
osxphotos TEXT,
exportdb TEXT
); """,
""" CREATE TABLE IF NOT EXISTS about (
id INTEGER PRIMARY KEY,
about TEXT
);""",
""" CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY,
filepath TEXT NOT NULL,
filepath_normalized TEXT NOT NULL,
uuid TEXT,
orig_mode INTEGER,
orig_size INTEGER,
orig_mtime REAL,
exif_mode INTEGER,
exif_size INTEGER,
exif_mtime REAL
); """,
""" CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY,
datetime TEXT,
python_path TEXT,
script_name TEXT,
args TEXT,
cwd TEXT
); """,
""" CREATE TABLE IF NOT EXISTS info (
id INTEGER PRIMARY KEY,
uuid text NOT NULL,
json_info JSON
); """,
""" CREATE TABLE IF NOT EXISTS exifdata (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
json_exifdata JSON
); """,
""" CREATE TABLE IF NOT EXISTS edited (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
""" CREATE TABLE IF NOT EXISTS converted (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
""" CREATE TABLE IF NOT EXISTS sidecar (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
sidecar_data TEXT,
mode INTEGER,
size INTEGER,
mtime REAL
); """,
""" CREATE TABLE IF NOT EXISTS detected_text (
id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL,
text_data JSON
); """,
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
]
# create the tables if needed
try:
c = conn.cursor()
for cmd in sql_commands.values():
for cmd in sql_commands:
c.execute(cmd)
c.execute(
"INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);",
@@ -772,6 +823,19 @@ class ExportDB(ExportDB_ABC):
except Error as e:
logging.warning(e)
# perform needed migrations
if version[1] < OSXPHOTOS_EXPORTDB_VERSION_MIGRATE_FILEPATH:
self._migrate_normalized_filepath(conn)
if version[1] < OSXPHOTOS_EXPORTDB_VERSION:
try:
c = conn.cursor()
# add metadata column to files to support --force-update
c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;")
conn.commit()
except Error as e:
logging.warning(e)
def __del__(self):
"""ensure the database connection is closed"""
try:
@@ -810,6 +874,28 @@ class ExportDB(ExportDB_ABC):
"""Fix all filepath_normalized columns for unicode normalization"""
# Prior to database version 4.3, filepath_normalized was not normalized for unicode
c = conn.cursor()
migration_sql = [
""" CREATE TABLE IF NOT EXISTS files_migrate (
id INTEGER PRIMARY KEY,
filepath TEXT NOT NULL,
filepath_normalized TEXT NOT NULL,
uuid TEXT,
orig_mode INTEGER,
orig_size INTEGER,
orig_mtime REAL,
exif_mode INTEGER,
exif_size INTEGER,
exif_mtime REAL,
UNIQUE(filepath_normalized)
); """,
""" INSERT INTO files_migrate SELECT * FROM files;""",
""" DROP TABLE files;""",
""" ALTER TABLE files_migrate RENAME TO files;""",
]
for sql in migration_sql:
c.execute(sql)
conn.commit()
for table in ["converted", "edited", "exifdata", "files", "sidecar"]:
old_values = c.execute(
f"SELECT filepath_normalized, id FROM {table}"
@@ -848,7 +934,7 @@ class ExportDBInMemory(ExportDB):
conn = self._get_db_connection()
if not conn:
raise Exception("Error getting connection to in-memory database")
self._create_db_tables(conn)
self._create_or_migrate_db_tables(conn)
self.was_created = True
self.was_upgraded = ()
else:
@@ -871,7 +957,7 @@ class ExportDBInMemory(ExportDB):
self.was_created = False
_, exportdb_ver = self._get_database_version(conn)
if exportdb_ver < OSXPHOTOS_EXPORTDB_VERSION:
self._create_db_tables(conn)
self._create_or_migrate_db_tables(conn)
self.was_upgraded = (exportdb_ver, OSXPHOTOS_EXPORTDB_VERSION)
else:
self.was_upgraded = ()

57
osxphotos/photoexporter.py
View File

@@ -83,6 +83,7 @@ class ExportOptions:
export_as_hardlink: (bool, default=False): if True, will hardlink files instead of copying them
export_db: (ExportDB_ABC): instance of a class that conforms to ExportDB_ABC with methods for getting/setting data related to exported files to compare update state
fileutil: (FileUtilABC): class that conforms to FileUtilABC with various file utilities
force_update: (bool, default=False): if True, will export photo if any metadata has changed but export otherwise would not be triggered (e.g. metadata changed but not using exiftool)
ignore_date_modified (bool): for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
ignore_signature (bool, default=False): ignore file signature when used with update (look only at filename)
increment (bool, default=True): if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists
@@ -128,6 +129,7 @@ class ExportOptions:
export_as_hardlink: bool = False
export_db: Optional[ExportDB_ABC] = None
fileutil: Optional[FileUtil] = None
force_update: bool = False
ignore_date_modified: bool = False
ignore_signature: bool = False
increment: bool = True
@@ -523,7 +525,7 @@ class PhotoExporter:
# the export directory
preview_name = (
preview_name
if options.overwrite or options.update
if any([options.overwrite, options.update, options.force_update])
else pathlib.Path(increment_filename(preview_name))
)
all_results += self._export_photo(
@@ -589,7 +591,7 @@ class PhotoExporter:
# if overwrite==False and #increment==False, export should fail if file exists
if dest.exists() and not any(
[options.increment, options.update, options.overwrite]
[options.increment, options.update, options.force_update, options.overwrite]
):
raise FileExistsError(
f"destination exists ({dest}); overwrite={options.overwrite}, increment={options.increment}"
@@ -601,11 +603,13 @@ class PhotoExporter:
# e.g. exporting sidecar for file1.png and file1.jpeg
# if file1.png exists and exporting file1.jpeg,
# dest will be file1 (1).jpeg even though file1.jpeg doesn't exist to prevent sidecar collision
if options.increment and not options.update and not options.overwrite:
if options.increment and not any(
[options.update, options.force_update, options.overwrite]
):
return pathlib.Path(increment_filename(dest))
# if update and file exists, need to check to see if it's the write file by checking export db
if options.update and dest.exists() and src:
if (options.update or options.force_update) and dest.exists() and src:
export_db = options.export_db
fileutil = options.fileutil
# destination exists, check to see if destination is the right UUID
@@ -735,7 +739,7 @@ class PhotoExporter:
# export live_photo .mov file?
live_photo = bool(options.live_photo and self.photo.live_photo)
overwrite = options.overwrite or options.update
overwrite = any([options.overwrite, options.update, options.force_update])
# figure out which photo version to request
if options.edited or self.photo.shared:
@@ -843,7 +847,7 @@ class PhotoExporter:
# export live_photo .mov file?
live_photo = bool(options.live_photo and self.photo.live_photo)
overwrite = options.overwrite or options.update
overwrite = any([options.overwrite, options.update, options.force_update])
edited_version = options.edited or self.photo.shared
# shared photos (in shared albums) show up as not having adjustments (not edited)
# but Photos is unable to export the "original" as only a jpeg copy is shared in iCloud
@@ -995,16 +999,11 @@ class PhotoExporter:
fileutil = options.fileutil
export_db = options.export_db
if options.update: # updating
if options.update or options.force_update: # updating
cmp_touch, cmp_orig = False, False
if dest_exists:
# update, destination exists, but we might not need to replace it...
if options.ignore_signature:
cmp_orig = True
cmp_touch = fileutil.cmp(
src, dest, mtime1=int(self.photo.date.timestamp())
)
elif options.exiftool:
if options.exiftool:
sig_exif = export_db.get_stat_exif_for_file(dest_str)
cmp_orig = fileutil.cmp_file_sig(dest_str, sig_exif)
if cmp_orig:
@@ -1026,10 +1025,17 @@ class PhotoExporter:
)
cmp_touch = fileutil.cmp_file_sig(dest_str, sig_converted)
else:
cmp_orig = fileutil.cmp(src, dest)
cmp_orig = options.ignore_signature or fileutil.cmp(src, dest)
cmp_touch = fileutil.cmp(
src, dest, mtime1=int(self.photo.date.timestamp())
)
if options.force_update:
# need to also check the photo's metadata to that in the database
# and if anything changed, we need to update the file
# ony the hex digest of the metadata is stored in the database
cmp_orig = hexdigest(
self.photo.json()
) == export_db.get_metadata_for_file(dest_str)
sig_cmp = cmp_touch if options.touch_file else cmp_orig
@@ -1043,7 +1049,7 @@ class PhotoExporter:
if sig_edited != (None, None, None)
else False
)
sig_cmp = sig_cmp and cmp_edited
sig_cmp = sig_cmp and (options.force_update or cmp_edited)
if (options.export_as_hardlink and dest.samefile(src)) or (
not options.export_as_hardlink
@@ -1086,7 +1092,9 @@ class PhotoExporter:
edited_stat = (
fileutil.file_sig(src) if options.edited else (None, None, None)
)
if dest_exists and (options.update or options.overwrite):
if dest_exists and any(
[options.overwrite, options.update, options.force_update]
):
# need to remove the destination first
try:
fileutil.unlink(dest)
@@ -1129,13 +1137,15 @@ class PhotoExporter:
f"Error copying file {src} to {dest_str}: {e} ({lineno(__file__)})"
) from e
json_info = self.photo.json()
export_db.set_data(
filename=dest_str,
uuid=self.photo.uuid,
orig_stat=fileutil.file_sig(dest_str),
converted_stat=converted_stat,
edited_stat=edited_stat,
info_json=self.photo.json(),
info_json=json_info,
metadata=hexdigest(json_info),
)
return ExportResults(
@@ -1235,10 +1245,13 @@ class PhotoExporter:
sidecar_filename
)
write_sidecar = (
not options.update
or (options.update and not sidecar_filename.exists())
not (options.update or options.force_update)
or (
options.update
(options.update or options.force_update)
and not sidecar_filename.exists()
)
or (
(options.update or options.force_update)
and (sidecar_digest != old_sidecar_digest)
or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig)
)
@@ -1333,8 +1346,8 @@ class PhotoExporter:
def _should_run_exiftool(self, dest, options: ExportOptions) -> bool:
"""Return True if exiftool should be run to update metadata"""
run_exiftool = not options.update
if options.update:
run_exiftool = not (options.update or options.force_update)
if options.update or options.force_update:
files_are_different = False
old_data = options.export_db.get_exifdata_for_file(dest)
if old_data is not None:

3
osxphotos/photoinfo.py
View File

@@ -1728,7 +1728,8 @@ class PhotoInfo:
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()
return json.dumps(self.asdict(), sort_keys=True, default=default)
dict_data = self.asdict()
return json.dumps(dict_data, sort_keys=True, default=default)
def __eq__(self, other):
"""Compare two PhotoInfo objects for equality"""