forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added errors to export database, --update-errors to export, #872 (#874)

Browse Source
This commit is contained in:
Rhet Turnbull 2022-12-18 14:16:38 -08:00 committed by GitHub
parent 8b9af7be67
commit b2b814954b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 335 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
osxphotos/cli/export.py
View File

@ -124,6 +124,17 @@ from .verbose import get_verbose_console, time_stamp, verbose_print
"if their metadata has changed even if this would not otherwise trigger an export. "
"See also --update and notes below on export and --update.",
)
@click.option(
"--update-errors",
is_flag=True,
help="Update files that were previously exported but produced errors during export. "
"For example, if a file produced an error with --exiftool due to bad metadata, "
"this option will re-export the file and attempt to write the metadata again "
"when used with --exiftool and --update. "
"Without --update-errors, photos that were successfully exported but generated "
"an error or warning during export will not be re-attempted if metadata has not changed. "
"Must be used with --update.",
)
@click.option(
"--ignore-signature",
is_flag=True,
@ -888,6 +899,7 @@ def export(
to_time,
touch_file,
update,
update_errors,
use_photokit,
use_photos_export,
uti,
@ -1109,6 +1121,7 @@ def export(
to_time = cfg.to_time
touch_file = cfg.touch_file
update = cfg.update
update_errors = cfg.update_errors
use_photokit = cfg.use_photokit
use_photos_export = cfg.use_photos_export
uti = cfg.uti
@ -1148,9 +1161,10 @@ def export(
("hdr", "not_hdr"),
("hidden", "not_hidden"),
("in_album", "not_in_album"),
("is_reference", "not_reference"),
("keyword", "no_keyword"),
("live", "not_live"),
("location", "no_location"),
("keyword", "no_keyword"),
("only_photos", "only_movies"),
("panorama", "not_panorama"),
("place", "no_place"),
@ -1162,19 +1176,19 @@ def export(
("slow_mo", "not_slow_mo"),
("time_lapse", "not_time_lapse"),
("title", "no_title"),
("is_reference", "not_reference"),
]
dependent_options = [
("append", ("report")),
("exiftool_merge_keywords", ("exiftool", "sidecar")),
("exiftool_merge_persons", ("exiftool", "sidecar")),
("favorite_rating", ("exiftool", "sidecar")),
("exiftool_option", ("exiftool")),
("favorite_rating", ("exiftool", "sidecar")),
("ignore_signature", ("update", "force_update")),
("jpeg_quality", ("convert_to_jpeg")),
("keep", ("cleanup")),
("missing", ("download_missing", "use_photos_export")),
("only_new", ("update", "force_update")),
("append", ("report")),
("update_errors", ("update")),
]
try:
cfg.validate(exclusive=exclusive_options, dependent=dependent_options, cli=True)
@ -1563,6 +1577,7 @@ def export(
strip=strip,
touch_file=touch_file,
update=update,
update_errors=update_errors,
use_photokit=use_photokit,
use_photos_export=use_photos_export,
verbose_=verbose_,
@ -1849,6 +1864,7 @@ def export_photo(
photo_num=1,
num_photos=1,
tmpdir=None,
update_errors=False,
):
"""Helper function for export that does the actual export
@ -1895,6 +1911,7 @@ def export_photo(
skip_original_if_edited: bool; if True does not export original if photo has been edited
touch_file: bool; sets file's modification time to match photo date
update: bool, only export updated photos
update_errors: bool, attempt to re-export photos that previously produced errors even if they otherwise would not be exported
use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing
verbose_: callable for verbose output
tmpdir: optional str; temporary directory to use for export
@ -2060,6 +2077,7 @@ def export_photo(
sidecar_flags=sidecar_flags,
touch_file=touch_file,
update=update,
update_errors=update_errors,
use_photos_export=use_photos_export,
use_photokit=use_photokit,
verbose_=verbose_,
@ -2175,6 +2193,7 @@ def export_photo(
sidecar_flags=sidecar_flags if not export_original else 0,
touch_file=touch_file,
update=update,
update_errors=update_errors,
use_photos_export=use_photos_export,
use_photokit=use_photokit,
verbose_=verbose_,
@ -2261,6 +2280,7 @@ def export_photo_to_directory(
sidecar_flags,
touch_file,
update,
update_errors,
use_photos_export,
use_photokit,
verbose_,
@ -2298,8 +2318,8 @@ def export_photo_to_directory(
download_missing=download_missing,
dry_run=dry_run,
edited=edited,
exiftool_flags=exiftool_option,
exiftool=exiftool,
exiftool_flags=exiftool_option,
export_as_hardlink=export_as_hardlink,
export_db=export_db,
favorite_rating=favorite_rating,
@ -2314,22 +2334,23 @@ def export_photo_to_directory(
merge_exif_keywords=exiftool_merge_keywords,
merge_exif_persons=exiftool_merge_persons,
overwrite=overwrite,
preview_suffix=preview_suffix,
preview=export_preview or (missing and preview_if_missing),
preview_suffix=preview_suffix,
raw_photo=export_raw,
render_options=render_options,
replace_keywords=replace_keywords,
sidecar_drop_ext=sidecar_drop_ext,
rich=True,
sidecar=sidecar_flags,
sidecar_drop_ext=sidecar_drop_ext,
tmpdir=tmpdir,
touch_file=touch_file,
update=update,
update_errors=update_errors,
use_albums_as_keywords=album_keyword,
use_persons_as_keywords=person_keyword,
use_photokit=use_photokit,
use_photos_export=use_photos_export,
verbose=verbose_,
tmpdir=tmpdir,
rich=True,
)
exporter = PhotoExporter(photo)
export_results = exporter.export(

165
osxphotos/cli/exportdb.py
View File

@ -16,6 +16,7 @@ from osxphotos.export_db import (
)
from osxphotos.export_db_utils import (
export_db_check_signatures,
export_db_get_errors,
export_db_get_last_run,
export_db_get_version,
export_db_save_config_to_file,
@ -25,10 +26,18 @@ from osxphotos.export_db_utils import (
)
from osxphotos.utils import pluralize
from .click_rich_echo import (
rich_click_echo,
rich_echo,
rich_echo_error,
set_rich_console,
set_rich_theme,
)
from .color_themes import get_theme
from .export import render_and_validate_report
from .param_types import TemplateString
from .report_writer import report_writer_factory
from .verbose import verbose_print
from .verbose import get_verbose_console, verbose_print
@click.command(name="exportdb")
@ -65,6 +74,16 @@ from .verbose import verbose_print
nargs=1,
help="Print information about FILE_PATH contained in the database.",
)
@click.option(
"--errors",
is_flag=True,
help="Print list of files that had warnings/errors on export (from all runs).",
)
@click.option(
"--last-errors",
is_flag=True,
help="Print list of files that had warnings/errors on last export run.",
)
@click.option(
"--uuid-files",
metavar="UUID",
@ -145,6 +164,8 @@ def exportdb(
export_db,
export_dir,
info,
errors,
last_errors,
last_run,
migrate,
report,
@ -161,13 +182,18 @@ def exportdb(
version,
):
"""Utilities for working with the osxphotos export database"""
verbose_ = verbose_print(verbose, rich=True)
color_theme = get_theme()
verbose_ = verbose_print(
verbose, timestamp=False, rich=True, theme=color_theme, highlight=False
)
# set console for rich_echo to be same as for verbose_
set_rich_console(get_verbose_console(theme=color_theme))
set_rich_theme(color_theme)
# validate options and args
if append and not report:
print(
"[red]Error: --append requires --report; ee --help for more information.[/]",
rich_echo(
"[error]Error: --append requires --report; ee --help for more information.[/]",
file=sys.stderr,
)
sys.exit(1)
@ -177,8 +203,8 @@ def exportdb(
# assume it's the export folder
export_db = export_db / OSXPHOTOS_EXPORT_DB
if not export_db.is_file():
print(
f"[red]Error: {OSXPHOTOS_EXPORT_DB} missing from {export_db.parent}[/red]"
rich_echo(
f"[error]Error: {OSXPHOTOS_EXPORT_DB} missing from {export_db.parent}[/error]"
)
sys.exit(1)
@ -203,7 +229,7 @@ def exportdb(
]
]
if sum(sub_commands) > 1:
print("[red]Only a single sub-command may be specified at a time[/red]")
rich_echo("[error]Only a single sub-command may be specified at a time[/error]")
sys.exit(1)
# process sub-commands
@ -212,11 +238,13 @@ def exportdb(
try:
osxphotos_ver, export_db_ver = export_db_get_version(export_db)
except Exception as e:
print(f"[red]Error: could not read version from {export_db}: {e}[/red]")
rich_echo(
f"[error]Error: could not read version from {export_db}: {e}[/error]"
)
sys.exit(1)
else:
print(
f"osxphotos version: {osxphotos_ver}, export database version: {export_db_ver}"
rich_echo(
f"osxphotos version: [num]{osxphotos_ver}[/], export database version: [num]{export_db_ver}[/]"
)
sys.exit(0)
@ -225,11 +253,11 @@ def exportdb(
start_size = pathlib.Path(export_db).stat().st_size
export_db_vacuum(export_db)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(
f"Vacuumed {export_db}! {start_size} bytes -> {pathlib.Path(export_db).stat().st_size} bytes"
rich_echo(
f"Vacuumed {export_db}! [num]{start_size}[/] bytes -> [num]{pathlib.Path(export_db).stat().st_size}[/] bytes"
)
sys.exit(0)
@ -239,31 +267,33 @@ def exportdb(
export_db, export_dir, verbose_, dry_run
)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(f"Done. Updated {updated} files, skipped {skipped} files.")
rich_echo(
f"Done. Updated [num]{updated}[/] files, skipped [num]{skipped}[/] files."
)
sys.exit(0)
if last_run:
try:
last_run_info = export_db_get_last_run(export_db)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(f"last run at {last_run_info[0]}:")
print(f"osxphotos {last_run_info[1]}")
rich_echo(f"last run at [time]{last_run_info[0]}:")
rich_echo(f"osxphotos {last_run_info[1]}")
sys.exit(0)
if save_config:
try:
export_db_save_config_to_file(export_db, save_config)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(f"Saved configuration to {save_config}")
rich_echo(f"Saved configuration to [filepath]{save_config}")
sys.exit(0)
if check_signatures:
@ -272,11 +302,12 @@ def exportdb(
export_db, export_dir, verbose_=verbose_
)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(
f"Done. Found {matched} matching signatures and {notmatched} signatures that don't match. Skipped {skipped} missing files."
rich_echo(
f"Done. Found [num]{matched}[/] matching signatures and [num]{notmatched}[/] signatures that don't match. "
f"Skipped [num]{skipped}[/] missing files."
)
sys.exit(0)
@ -286,11 +317,12 @@ def exportdb(
export_db, export_dir, verbose_=verbose_, dry_run=dry_run
)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
print(
f"Done. Touched {touched} files, skipped {not_touched} up to date files, skipped {skipped} missing files."
rich_echo(
f"Done. Touched [num]{touched}[/] files, skipped [num]{not_touched}[/] up to date files, "
f"skipped [num]{skipped}[/] missing files."
)
sys.exit(0)
@ -299,28 +331,63 @@ def exportdb(
try:
info_rec = exportdb.get_file_record(info)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
if info_rec:
# use rich print as rich_echo doesn't highlight json
print(info_rec.json(indent=2))
else:
print(f"[red]File '{info}' not found in export database[/red]")
rich_echo(f"[error]File '{info}' not found in export database[/error]")
sys.exit(0)
if errors:
# list errors
try:
error_list = export_db_get_errors(export_db)
except Exception as e:
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
if error_list:
for error in error_list:
rich_echo(error)
else:
rich_echo("No errors found")
sys.exit(0)
if last_errors:
exportdb = ExportDB(export_db, export_dir)
if export_results := exportdb.get_export_results(0):
for error in [
*export_results.error,
*export_results.exiftool_error,
*export_results.exiftool_warning,
]:
rich_click_echo(
f"[filepath]{error[0]}[/], [time]{export_results.datetime}[/], [error]{error[1]}[/]"
)
sys.exit(0)
else:
rich_echo_error("[error]Results from last run not found in database[/]")
sys.exit(1)
if uuid_info:
# get photoinfo record for a uuid
exportdb = ExportDB(export_db, export_dir)
try:
info_rec = exportdb.get_photoinfo_for_uuid(uuid_info)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
if info_rec:
# use rich print as rich_echo doesn't highlight json
print(json.dumps(json.loads(info_rec), sort_keys=True, indent=2))
else:
print(f"[red]UUID '{uuid_info}' not found in export database[/red]")
rich_echo(
f"[error]UUID '{uuid_info}' not found in export database[/error]"
)
sys.exit(0)
if uuid_files:
@ -329,32 +396,38 @@ def exportdb(
try:
file_list = exportdb.get_files_for_uuid(uuid_files)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
if file_list:
for f in file_list:
print(f)
rich_echo(f"[filepath]{f}[/]")
else:
print(f"[red]UUID '{uuid_files}' not found in export database[/red]")
rich_echo(
f"[error]UUID '{uuid_files}' not found in export database[/error]"
)
sys.exit(0)
if delete_uuid:
# delete a uuid from the export database
exportdb = ExportDB(export_db, export_dir)
for uuid in delete_uuid:
print(f"Deleting uuid {uuid} from database.")
rich_echo(f"Deleting uuid [uuid]{uuid}[/] from database.")
count = exportdb.delete_data_for_uuid(uuid)
print(f"Deleted {count} {pluralize(count, 'record', 'records')}.")
rich_echo(
f"Deleted [num]{count}[/] {pluralize(count, 'record', 'records')}."
)
sys.exit(0)
if delete_file:
# delete information associated with a file from the export database
exportdb = ExportDB(export_db, export_dir)
for filepath in delete_file:
print(f"Deleting file {filepath} from database.")
rich_echo(f"Deleting file [filepath]{filepath}[/] from database.")
count = exportdb.delete_data_for_filepath(filepath)
print(f"Deleted {count} {pluralize(count, 'record', 'records')}.")
rich_echo(
f"Deleted [num]{count}[/] {pluralize(count, 'record', 'records')}."
)
sys.exit(0)
if report:
@ -363,27 +436,27 @@ def exportdb(
report_filename = render_and_validate_report(report_template, "", export_dir)
export_results = exportdb.get_export_results(run_id)
if not export_results:
print(f"[red]No report results found for run ID {run_id}[/red]")
rich_echo(f"[error]No report results found for run ID {run_id}[/error]")
sys.exit(1)
try:
report_writer = report_writer_factory(report_filename, append=append)
except ValueError as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
report_writer.write(export_results)
report_writer.close()
print(f"Wrote report to {report_filename}")
rich_echo(f"Wrote report to [filepath]{report_filename}[/]")
sys.exit(0)
if migrate:
exportdb = ExportDB(export_db, export_dir)
if upgraded := exportdb.was_upgraded:
print(
f"Migrated export database {export_db} from version {upgraded[0]} to {upgraded[1]}"
rich_echo(
f"Migrated export database [filepath]{export_db}[/] from version [num]{upgraded[0]}[/] to [num]{upgraded[1]}[/]"
)
else:
print(
f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}"
rich_echo(
f"Export database [filepath]{export_db}[/] is already at latest version [num]{OSXPHOTOS_EXPORTDB_VERSION}[/]"
)
sys.exit(0)
@ -393,7 +466,7 @@ def exportdb(
c = exportdb._conn.cursor()
results = c.execute(sql)
except Exception as e:
print(f"[red]Error: {e}[/red]")
rich_echo(f"[error]Error: {e}[/error]")
sys.exit(1)
else:
for row in results:

49
osxphotos/export_db.py
View File

@ -1,10 +1,10 @@
""" Helper class for managing database used by PhotoExporter for tracking state of exports and updates """
from __future__ import annotations
import datetime
import gzip
import json
import logging
import os
import os.path
import pathlib
@ -32,7 +32,7 @@ __all__ = [
"ExportDBTemp",
]
OSXPHOTOS_EXPORTDB_VERSION = "7.1"
OSXPHOTOS_EXPORTDB_VERSION = "8.0"
OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
# max retry attempts for methods which use tenacity.retry
@ -532,6 +532,10 @@ class ExportDB:
# add timestamp to export_data
self._migrate_7_0_to_7_1(conn)
if version[1] < "8.0":
# add error to export_data
self._migrate_7_1_to_8_0(conn)
conn.execute("VACUUM;")
conn.commit()
@ -739,6 +743,7 @@ class ExportDB:
conn.commit()
def _migrate_7_0_to_7_1(self, conn):
"""Add timestamp column to export_data table and triggers to update it on insert and update."""
c = conn.cursor()
# timestamp column should not exist but this prevents error if migration is run on an already migrated database
# reference #794
@ -767,6 +772,16 @@ class ExportDB:
)
conn.commit()
def _migrate_7_1_to_8_0(self, conn):
"""Add error column to export_data table"""
c = conn.cursor()
results = c.execute(
"SELECT COUNT(*) FROM pragma_table_info('export_data') WHERE name='error';"
).fetchone()
if results[0] == 0:
c.execute("""ALTER TABLE export_data ADD COLUMN error JSON;""")
conn.commit()
def _perform_db_maintenace(self, conn):
"""Perform database maintenance"""
c = conn.cursor()
@ -1133,6 +1148,35 @@ class ExportRecord:
f"No timestamp found in database for {self._filepath_normalized}"
)
@property
def error(self) -> dict[str, Any] | None:
"""Return error value"""
conn = self._conn
c = conn.cursor()
if row := c.execute(
"SELECT error FROM export_data WHERE filepath_normalized = ?;",
(self._filepath_normalized,),
).fetchone():
return json.loads(row[0]) if row[0] else None
raise ValueError(f"No error found in database for {self._filepath_normalized}")
@error.setter
def error(self, value: dict[str, Any] | None):
"""Set error value"""
conn = self._conn
c = conn.cursor()
if value is None:
value = ""
# use default=str because some of the values are Path objects
error = json.dumps(value, default=str)
c.execute(
"UPDATE export_data SET error = ? WHERE filepath_normalized = ?;",
(error, self._filepath_normalized),
)
if not self._context_manager:
conn.commit()
def asdict(self):
"""Return dict of self"""
exifdata = json.loads(self.exifdata) if self.exifdata else None
@ -1147,6 +1191,7 @@ class ExportRecord:
"dest_sig": self.dest_sig,
"export_options": self.export_options,
"exifdata": exifdata,
"error": self.error,
"photoinfo": photoinfo,
}

70
osxphotos/export_db_utils.py
View File

@ -20,6 +20,7 @@ from .utils import noop
__all__ = [
"export_db_check_signatures",
"export_db_get_errors",
"export_db_get_last_run",
"export_db_get_version",
"export_db_save_config_to_file",
@ -40,10 +41,9 @@ def export_db_get_version(
"""returns version from export database as tuple of (osxphotos version, export_db version)"""
conn = sqlite3.connect(str(dbfile))
c = conn.cursor()
row = c.execute(
if row := c.execute(
"SELECT osxphotos, exportdb FROM version ORDER BY id DESC LIMIT 1;"
).fetchone()
if row:
).fetchone():
return (row[0], row[1])
return (None, None)
@ -80,11 +80,13 @@ def export_db_update_signatures(
filepath = export_dir / filepath
if not os.path.exists(filepath):
skipped += 1
verbose_(f"[dark_orange]Skipping missing file[/dark_orange]: '{filepath}'")
verbose_(
f"[dark_orange]Skipping missing file[/dark_orange]: '[filepath]{filepath}[/]'"
)
continue
updated += 1
file_sig = fileutil.file_sig(filepath)
verbose_(f"[green]Updating signature for[/green]: '{filepath}'")
verbose_(f"[green]Updating signature for[/green]: '[filepath]{filepath}[/]'")
if not dry_run:
c.execute(
"UPDATE export_data SET dest_mode = ?, dest_size = ?, dest_mtime = ? WHERE filepath_normalized = ?;",
@ -103,14 +105,29 @@ def export_db_get_last_run(
"""Get last run from export database"""
conn = sqlite3.connect(str(export_db))
c = conn.cursor()
row = c.execute(
if row := c.execute(
"SELECT datetime, args FROM runs ORDER BY id DESC LIMIT 1;"
).fetchone()
if row:
).fetchone():
return row[0], row[1]
return None, None
def export_db_get_errors(
export_db: Union[str, pathlib.Path]
) -> Tuple[Optional[str], Optional[str]]:
"""Get errors from export database"""
conn = sqlite3.connect(str(export_db))
c = conn.cursor()
results = c.execute(
"SELECT filepath, uuid, timestamp, error FROM export_data WHERE error is not null ORDER BY timestamp DESC;"
).fetchall()
results = [
f"[filepath]{row[0]}[/], [uuid]{row[1]}[/], [time]{row[2]}[/], [error]{row[3]}[/]"
for row in results
]
return results
def export_db_save_config_to_file(
export_db: Union[str, pathlib.Path], config_file: Union[str, pathlib.Path]
) -> None:
@ -138,9 +155,11 @@ def export_db_get_config(
conn = sqlite3.connect(str(export_db))
c = conn.cursor()
row = c.execute("SELECT config FROM config ORDER BY id DESC LIMIT 1;").fetchone()
if not row:
return ValueError("No config found in export_db")
return config.load_from_str(row[0], override=override)
return (
config.load_from_str(row[0], override=override)
if row
else ValueError("No config found in export_db")
)
def export_db_check_signatures(
@ -168,16 +187,20 @@ def export_db_check_signatures(
filepath = export_dir / filepath
if not filepath.exists():
skipped += 1
verbose_(f"[dark_orange]Skipping missing file[/dark_orange]: '{filepath}'")
verbose_(
f"[dark_orange]Skipping missing file[/dark_orange]: '[filepath]{filepath}[/]'"
)
continue
file_sig = fileutil.file_sig(filepath)
file_rec = exportdb.get_file_record(filepath)
if file_rec.dest_sig == file_sig:
matched += 1
verbose_(f"[green]Signatures matched[/green]: '{filepath}'")
verbose_(f"[green]Signatures matched[/green]: '[filepath]{filepath}[/]'")
else:
notmatched += 1
verbose_(f"[deep_pink3]Signatures do not match[/deep_pink3]: '{filepath}'")
verbose_(
f"[deep_pink3]Signatures do not match[/deep_pink3]: '[filepath]{filepath}[/]'"
)
return (matched, notmatched, skipped)
@ -196,8 +219,7 @@ def export_db_touch_files(
# open and close exportdb to ensure it gets migrated
exportdb = ExportDB(dbfile, export_dir)
upgraded = exportdb.was_upgraded
if upgraded:
if upgraded := exportdb.was_upgraded:
verbose_(
f"Upgraded export database {dbfile} from version {upgraded[0]} to {upgraded[1]}"
)
@ -205,9 +227,9 @@ def export_db_touch_files(
conn = sqlite3.connect(str(dbfile))
c = conn.cursor()
# get most recent config
row = c.execute("SELECT config FROM config ORDER BY id DESC LIMIT 1;").fetchone()
if row:
if row := c.execute(
"SELECT config FROM config ORDER BY id DESC LIMIT 1;"
).fetchone():
config = toml.loads(row[0])
try:
photos_db_path = config["export"].get("db", None)
@ -237,7 +259,7 @@ def export_db_touch_files(
if not filepath.exists():
skipped += 1
verbose_(
f"[dark_orange]Skipping missing file (not in export directory)[/dark_orange]: '{filepath}'"
f"[dark_orange]Skipping missing file (not in export directory)[/dark_orange]: '[filepath]{filepath}[/]'"
)
continue
@ -245,7 +267,7 @@ def export_db_touch_files(
if not photo:
skipped += 1
verbose_(
f"[dark_orange]Skipping missing photo (did not find in Photos Library)[/dark_orange]: '{filepath}' ({uuid})"
f"[dark_orange]Skipping missing photo (did not find in Photos Library)[/dark_orange]: '[filepath]{filepath}[/]' ([uuid]{uuid}[/])"
)
continue
@ -255,14 +277,14 @@ def export_db_touch_files(
if mtime == ts:
not_touched += 1
verbose_(
f"[green]Skipping file (timestamp matches)[/green]: '{filepath}' [dodger_blue1]{isotime_from_ts(ts)} ({ts})[/dodger_blue1]"
f"[green]Skipping file (timestamp matches)[/green]: '[filepath]{filepath}[/]' [time]{isotime_from_ts(ts)} ({ts})[/time]"
)
continue
touched += 1
verbose_(
f"[deep_pink3]Touching file[/deep_pink3]: '{filepath}' "
f"[dodger_blue1]{isotime_from_ts(mtime)} ({mtime}) -> {isotime_from_ts(ts)} ({ts})[/dodger_blue1]"
f"[deep_pink3]Touching file[/deep_pink3]: '[filepath]{filepath}[/]' "
f"[time]{isotime_from_ts(mtime)} ({mtime}) -> {isotime_from_ts(ts)} ({ts})[/time]"
)
if not dry_run:

29
osxphotos/photoexporter.py
View File

@ -76,6 +76,7 @@ class ShouldUpdate(Enum):
EXIFTOOL_DIFFERENT = 6
EDITED_SIG_DIFFERENT = 7
DIGEST_DIFFERENT = 8
UPDATE_ERRORS = 9
class ExportError(Exception):
@ -130,6 +131,7 @@ class ExportOptions:
timeout (int, default=120): timeout in seconds used with use_photos_export
touch_file (bool, default=False): if True, sets file's modification time upon photo date
update (bool, default=False): if True export will run in update mode, that is, it will not export the photo if the current version already exists in the destination
update_errors (bool, default=False): if True photos that previously produced a warning or error will be re-exported; otherwise they will note be
use_albums_as_keywords (bool, default = False): if True, will include album names in keywords when exporting metadata with exiftool or sidecar
use_persons_as_keywords (bool, default = False): if True, will include person names in keywords when exporting metadata with exiftool or sidecar
use_photos_export (bool, default=False): if True will attempt to export photo via applescript interaction with Photos even if not missing (see also use_photokit, download_missing)
@ -176,6 +178,7 @@ class ExportOptions:
timeout: int = 120
touch_file: bool = False
update: bool = False
update_errors: bool = False
use_albums_as_keywords: bool = False
use_persons_as_keywords: bool = False
use_photokit: bool = False
@ -701,6 +704,10 @@ class PhotoExporter:
self, src: pathlib.Path, dest: pathlib.Path, options: ExportOptions
) -> t.Literal[True, False]:
"""Return True if photo should be updated, else False"""
# NOTE: The order of certain checks is important
# read the comments below to understand why before changing
export_db = options.export_db
fileutil = options.fileutil
@ -731,6 +738,14 @@ class PhotoExporter:
# as it'll be None and bit_flags will be an int
return ShouldUpdate.EXPORT_OPTIONS_DIFFERENT
if options.update_errors and file_record.error is not None:
# files that were exported but generated an error
# won't be updated unless --update-errors is specified
# for example, an exiftool error due to bad metadata
# that the user subsequently fixed should be updated; see #872
# this must be checked before exiftool which will return False if exif data matches
return ShouldUpdate.UPDATE_ERRORS
if options.exiftool:
current_exifdata = self.exiftool_json_sidecar(options=options)
rv = current_exifdata != file_record.exifdata
@ -1179,8 +1194,7 @@ class PhotoExporter:
# set data in the database
with export_db.create_or_get_file_record(dest_str, self.photo.uuid) as rec:
photoinfo = self.photo.json()
rec.photoinfo = photoinfo
rec.photoinfo = self.photo.json()
rec.export_options = options.bit_flags
# don't set src_sig as that is set above before any modifications by convert_to_jpeg or exiftool
if not options.ignore_signature:
@ -1190,6 +1204,17 @@ class PhotoExporter:
if self.photo.hexdigest != rec.digest:
results.metadata_changed = [dest_str]
rec.digest = self.photo.hexdigest
# save errors to the export database (#872)
if (
results.error
or exif_results.exiftool_error
or exif_results.exiftool_warning
):
rec.error = {
"error": results.error,
"exiftool_error": exif_results.exiftool_error,
"exiftool_warning": exif_results.exiftool_warning,
}
return results

84
tests/test_cli.py
View File

@ -2292,24 +2292,72 @@ def test_export_exiftool_favorite_rating():
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
for uuid in CLI_EXIFTOOL:
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_15_7),
".",
"-V",
"--exiftool",
"--uuid",
UUID_FAVORITE,
"--uuid",
UUID_NOT_FAVORITE,
"--favorite-rating",
],
)
assert result.exit_code == 0
assert ExifTool(FILE_FAVORITE).asdict()["XMP:Rating"] == 5
assert ExifTool(FILE_NOT_FAVORITE).asdict()["XMP:Rating"] == 0
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_15_7),
".",
"-V",
"--exiftool",
"--uuid",
UUID_FAVORITE,
"--uuid",
UUID_NOT_FAVORITE,
"--favorite-rating",
],
)
assert result.exit_code == 0
assert ExifTool(FILE_FAVORITE).asdict()["XMP:Rating"] == 5
assert ExifTool(FILE_NOT_FAVORITE).asdict()["XMP:Rating"] == 0
@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
def test_export_exiftool_update_errors():
"""Test export with --update-errors, #872"""
runner = CliRunner()
cwd = os.getcwd()
# first, normal export with --exiftool
# some of the files will have errors / warnings from exiftool
with runner.isolated_filesystem():
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_15_7),
".",
"-V",
"--exiftool",
],
)
# now run update; none of files should be updated
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_15_7),
".",
"-V",
"--exiftool",
"--update",
],
)
assert result.exit_code == 0
assert "updated: 0" in result.output
# now run update-errors; only files with errors should be updated
result = runner.invoke(
export,
[
os.path.join(cwd, PHOTOS_DB_15_7),
".",
"-V",
"--exiftool",
"--update",
"--update-errors",
],
)
assert result.exit_code == 0
assert "updated: 4" in result.output
def test_export_edited_suffix():