Added errors to export database, --update-errors to export, #872 (#874)

osxphotos/cli/export.py

@@ -124,6 +124,17 @@ from .verbose import get_verbose_console, time_stamp, verbose_print
     "if their metadata has changed even if this would not otherwise trigger an export. "
     "See also --update and notes below on export and --update.",
 )
+@click.option(
+    "--update-errors",
+    is_flag=True,
+    help="Update files that were previously exported but produced errors during export. "
+    "For example, if a file produced an error with --exiftool due to bad metadata, "
+    "this option will re-export the file and attempt to write the metadata again "
+    "when used with --exiftool and --update. "
+    "Without --update-errors, photos that were successfully exported but generated "
+    "an error or warning during export will not be re-attempted if metadata has not changed. "
+    "Must be used with --update.",
+)
 @click.option(
     "--ignore-signature",
     is_flag=True,
@@ -888,6 +899,7 @@ def export(
     to_time,
     touch_file,
     update,
+    update_errors,
     use_photokit,
     use_photos_export,
     uti,
@@ -1109,6 +1121,7 @@ def export(
         to_time = cfg.to_time
         touch_file = cfg.touch_file
         update = cfg.update
+        update_errors = cfg.update_errors
         use_photokit = cfg.use_photokit
         use_photos_export = cfg.use_photos_export
         uti = cfg.uti
@@ -1148,9 +1161,10 @@ def export(
         ("hdr", "not_hdr"),
         ("hidden", "not_hidden"),
         ("in_album", "not_in_album"),
+        ("is_reference", "not_reference"),
+        ("keyword", "no_keyword"),
         ("live", "not_live"),
         ("location", "no_location"),
-        ("keyword", "no_keyword"),
         ("only_photos", "only_movies"),
         ("panorama", "not_panorama"),
         ("place", "no_place"),
@@ -1162,19 +1176,19 @@ def export(
         ("slow_mo", "not_slow_mo"),
         ("time_lapse", "not_time_lapse"),
         ("title", "no_title"),
-        ("is_reference", "not_reference"),
     ]
     dependent_options = [
+        ("append", ("report")),
         ("exiftool_merge_keywords", ("exiftool", "sidecar")),
         ("exiftool_merge_persons", ("exiftool", "sidecar")),
-        ("favorite_rating", ("exiftool", "sidecar")),
         ("exiftool_option", ("exiftool")),
+        ("favorite_rating", ("exiftool", "sidecar")),
         ("ignore_signature", ("update", "force_update")),
         ("jpeg_quality", ("convert_to_jpeg")),
         ("keep", ("cleanup")),
         ("missing", ("download_missing", "use_photos_export")),
         ("only_new", ("update", "force_update")),
-        ("append", ("report")),
+        ("update_errors", ("update")),
     ]
     try:
         cfg.validate(exclusive=exclusive_options, dependent=dependent_options, cli=True)
@@ -1563,6 +1577,7 @@ def export(
                     strip=strip,
                     touch_file=touch_file,
                     update=update,
+                    update_errors=update_errors,
                     use_photokit=use_photokit,
                     use_photos_export=use_photos_export,
                     verbose_=verbose_,
@@ -1849,6 +1864,7 @@ def export_photo(
     photo_num=1,
     num_photos=1,
     tmpdir=None,
+    update_errors=False,
 ):
     """Helper function for export that does the actual export
 
@@ -1895,6 +1911,7 @@ def export_photo(
         skip_original_if_edited: bool; if True does not export original if photo has been edited
         touch_file: bool; sets file's modification time to match photo date
         update: bool, only export updated photos
+        update_errors: bool, attempt to re-export photos that previously produced errors even if they otherwise would not be exported
         use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing
         verbose_: callable for verbose output
         tmpdir: optional str; temporary directory to use for export
@@ -2060,6 +2077,7 @@ def export_photo(
                 sidecar_flags=sidecar_flags,
                 touch_file=touch_file,
                 update=update,
+                update_errors=update_errors,
                 use_photos_export=use_photos_export,
                 use_photokit=use_photokit,
                 verbose_=verbose_,
@@ -2175,6 +2193,7 @@ def export_photo(
                     sidecar_flags=sidecar_flags if not export_original else 0,
                     touch_file=touch_file,
                     update=update,
+                    update_errors=update_errors,
                     use_photos_export=use_photos_export,
                     use_photokit=use_photokit,
                     verbose_=verbose_,
@@ -2261,6 +2280,7 @@ def export_photo_to_directory(
     sidecar_flags,
     touch_file,
     update,
+    update_errors,
     use_photos_export,
     use_photokit,
     verbose_,
@@ -2298,8 +2318,8 @@ def export_photo_to_directory(
                 download_missing=download_missing,
                 dry_run=dry_run,
                 edited=edited,
-                exiftool_flags=exiftool_option,
                 exiftool=exiftool,
+                exiftool_flags=exiftool_option,
                 export_as_hardlink=export_as_hardlink,
                 export_db=export_db,
                 favorite_rating=favorite_rating,
@@ -2314,22 +2334,23 @@ def export_photo_to_directory(
                 merge_exif_keywords=exiftool_merge_keywords,
                 merge_exif_persons=exiftool_merge_persons,
                 overwrite=overwrite,
-                preview_suffix=preview_suffix,
                 preview=export_preview or (missing and preview_if_missing),
+                preview_suffix=preview_suffix,
                 raw_photo=export_raw,
                 render_options=render_options,
                 replace_keywords=replace_keywords,
-                sidecar_drop_ext=sidecar_drop_ext,
+                rich=True,
                 sidecar=sidecar_flags,
+                sidecar_drop_ext=sidecar_drop_ext,
+                tmpdir=tmpdir,
                 touch_file=touch_file,
                 update=update,
+                update_errors=update_errors,
                 use_albums_as_keywords=album_keyword,
                 use_persons_as_keywords=person_keyword,
                 use_photokit=use_photokit,
                 use_photos_export=use_photos_export,
                 verbose=verbose_,
-                tmpdir=tmpdir,
-                rich=True,
             )
             exporter = PhotoExporter(photo)
             export_results = exporter.export(

osxphotos/cli/exportdb.py

@@ -16,6 +16,7 @@ from osxphotos.export_db import (
 )
 from osxphotos.export_db_utils import (
     export_db_check_signatures,
+    export_db_get_errors,
     export_db_get_last_run,
     export_db_get_version,
     export_db_save_config_to_file,
@@ -25,10 +26,18 @@ from osxphotos.export_db_utils import (
 )
 from osxphotos.utils import pluralize
 
+from .click_rich_echo import (
+    rich_click_echo,
+    rich_echo,
+    rich_echo_error,
+    set_rich_console,
+    set_rich_theme,
+)
+from .color_themes import get_theme
 from .export import render_and_validate_report
 from .param_types import TemplateString
 from .report_writer import report_writer_factory
-from .verbose import verbose_print
+from .verbose import get_verbose_console, verbose_print
 
 
 @click.command(name="exportdb")
@@ -65,6 +74,16 @@ from .verbose import verbose_print
     nargs=1,
     help="Print information about FILE_PATH contained in the database.",
 )
+@click.option(
+    "--errors",
+    is_flag=True,
+    help="Print list of files that had warnings/errors on export (from all runs).",
+)
+@click.option(
+    "--last-errors",
+    is_flag=True,
+    help="Print list of files that had warnings/errors on last export run.",
+)
 @click.option(
     "--uuid-files",
     metavar="UUID",
@@ -145,6 +164,8 @@ def exportdb(
     export_db,
     export_dir,
     info,
+    errors,
+    last_errors,
     last_run,
     migrate,
     report,
@@ -161,13 +182,18 @@ def exportdb(
     version,
 ):
     """Utilities for working with the osxphotos export database"""
-
+    color_theme = get_theme()
-    verbose_ = verbose_print(verbose, rich=True)
+    verbose_ = verbose_print(
+        verbose, timestamp=False, rich=True, theme=color_theme, highlight=False
+    )
+    # set console for rich_echo to be same as for verbose_
+    set_rich_console(get_verbose_console(theme=color_theme))
+    set_rich_theme(color_theme)
 
     # validate options and args
     if append and not report:
-        print(
+        rich_echo(
-            "[red]Error: --append requires --report; ee --help for more information.[/]",
+            "[error]Error: --append requires --report; ee --help for more information.[/]",
             file=sys.stderr,
         )
         sys.exit(1)
@@ -177,8 +203,8 @@ def exportdb(
         # assume it's the export folder
         export_db = export_db / OSXPHOTOS_EXPORT_DB
         if not export_db.is_file():
-            print(
+            rich_echo(
-                f"[red]Error: {OSXPHOTOS_EXPORT_DB} missing from {export_db.parent}[/red]"
+                f"[error]Error: {OSXPHOTOS_EXPORT_DB} missing from {export_db.parent}[/error]"
             )
             sys.exit(1)
 
@@ -203,7 +229,7 @@ def exportdb(
         ]
     ]
     if sum(sub_commands) > 1:
-        print("[red]Only a single sub-command may be specified at a time[/red]")
+        rich_echo("[error]Only a single sub-command may be specified at a time[/error]")
         sys.exit(1)
 
     # process sub-commands
@@ -212,11 +238,13 @@ def exportdb(
         try:
             osxphotos_ver, export_db_ver = export_db_get_version(export_db)
         except Exception as e:
-            print(f"[red]Error: could not read version from {export_db}: {e}[/red]")
+            rich_echo(
+                f"[error]Error: could not read version from {export_db}: {e}[/error]"
+            )
             sys.exit(1)
         else:
-            print(
+            rich_echo(
-                f"osxphotos version: {osxphotos_ver}, export database version: {export_db_ver}"
+                f"osxphotos version: [num]{osxphotos_ver}[/], export database version: [num]{export_db_ver}[/]"
             )
         sys.exit(0)
 
@@ -225,11 +253,11 @@ def exportdb(
             start_size = pathlib.Path(export_db).stat().st_size
             export_db_vacuum(export_db)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(
+            rich_echo(
-                f"Vacuumed {export_db}! {start_size} bytes -> {pathlib.Path(export_db).stat().st_size} bytes"
+                f"Vacuumed {export_db}! [num]{start_size}[/] bytes -> [num]{pathlib.Path(export_db).stat().st_size}[/] bytes"
             )
             sys.exit(0)
 
@@ -239,31 +267,33 @@ def exportdb(
                 export_db, export_dir, verbose_, dry_run
             )
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(f"Done. Updated {updated} files, skipped {skipped} files.")
+            rich_echo(
+                f"Done. Updated [num]{updated}[/] files, skipped [num]{skipped}[/] files."
+            )
             sys.exit(0)
 
     if last_run:
         try:
             last_run_info = export_db_get_last_run(export_db)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(f"last run at {last_run_info[0]}:")
+            rich_echo(f"last run at [time]{last_run_info[0]}:")
-            print(f"osxphotos {last_run_info[1]}")
+            rich_echo(f"osxphotos {last_run_info[1]}")
             sys.exit(0)
 
     if save_config:
         try:
             export_db_save_config_to_file(export_db, save_config)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(f"Saved configuration to {save_config}")
+            rich_echo(f"Saved configuration to [filepath]{save_config}")
             sys.exit(0)
 
     if check_signatures:
@@ -272,11 +302,12 @@ def exportdb(
                 export_db, export_dir, verbose_=verbose_
             )
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(
+            rich_echo(
-                f"Done. Found {matched} matching signatures and {notmatched} signatures that don't match. Skipped {skipped} missing files."
+                f"Done. Found [num]{matched}[/] matching signatures and [num]{notmatched}[/] signatures that don't match. "
+                f"Skipped [num]{skipped}[/] missing files."
             )
             sys.exit(0)
 
@@ -286,11 +317,12 @@ def exportdb(
                 export_db, export_dir, verbose_=verbose_, dry_run=dry_run
             )
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
-            print(
+            rich_echo(
-                f"Done. Touched {touched} files, skipped {not_touched} up to date files, skipped {skipped} missing files."
+                f"Done. Touched [num]{touched}[/] files, skipped [num]{not_touched}[/] up to date files, "
+                f"skipped [num]{skipped}[/] missing files."
             )
             sys.exit(0)
 
@@ -299,28 +331,63 @@ def exportdb(
         try:
             info_rec = exportdb.get_file_record(info)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
             if info_rec:
+                # use rich print as rich_echo doesn't highlight json
                 print(info_rec.json(indent=2))
             else:
-                print(f"[red]File '{info}' not found in export database[/red]")
+                rich_echo(f"[error]File '{info}' not found in export database[/error]")
             sys.exit(0)
 
+    if errors:
+        # list errors
+        try:
+            error_list = export_db_get_errors(export_db)
+        except Exception as e:
+            rich_echo(f"[error]Error: {e}[/error]")
+            sys.exit(1)
+        else:
+            if error_list:
+                for error in error_list:
+                    rich_echo(error)
+            else:
+                rich_echo("No errors found")
+            sys.exit(0)
+
+    if last_errors:
+        exportdb = ExportDB(export_db, export_dir)
+        if export_results := exportdb.get_export_results(0):
+            for error in [
+                *export_results.error,
+                *export_results.exiftool_error,
+                *export_results.exiftool_warning,
+            ]:
+                rich_click_echo(
+                    f"[filepath]{error[0]}[/], [time]{export_results.datetime}[/], [error]{error[1]}[/]"
+                )
+            sys.exit(0)
+        else:
+            rich_echo_error("[error]Results from last run not found in database[/]")
+            sys.exit(1)
+
     if uuid_info:
         # get photoinfo record for a uuid
         exportdb = ExportDB(export_db, export_dir)
         try:
             info_rec = exportdb.get_photoinfo_for_uuid(uuid_info)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
             if info_rec:
+                # use rich print as rich_echo doesn't highlight json
                 print(json.dumps(json.loads(info_rec), sort_keys=True, indent=2))
             else:
-                print(f"[red]UUID '{uuid_info}' not found in export database[/red]")
+                rich_echo(
+                    f"[error]UUID '{uuid_info}' not found in export database[/error]"
+                )
             sys.exit(0)
 
     if uuid_files:
@@ -329,32 +396,38 @@ def exportdb(
         try:
             file_list = exportdb.get_files_for_uuid(uuid_files)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
             if file_list:
                 for f in file_list:
-                    print(f)
+                    rich_echo(f"[filepath]{f}[/]")
             else:
-                print(f"[red]UUID '{uuid_files}' not found in export database[/red]")
+                rich_echo(
+                    f"[error]UUID '{uuid_files}' not found in export database[/error]"
+                )
             sys.exit(0)
 
     if delete_uuid:
         # delete a uuid from the export database
         exportdb = ExportDB(export_db, export_dir)
         for uuid in delete_uuid:
-            print(f"Deleting uuid {uuid} from database.")
+            rich_echo(f"Deleting uuid [uuid]{uuid}[/] from database.")
             count = exportdb.delete_data_for_uuid(uuid)
-            print(f"Deleted {count} {pluralize(count, 'record', 'records')}.")
+            rich_echo(
+                f"Deleted [num]{count}[/] {pluralize(count, 'record', 'records')}."
+            )
         sys.exit(0)
 
     if delete_file:
         # delete information associated with a file from the export database
         exportdb = ExportDB(export_db, export_dir)
         for filepath in delete_file:
-            print(f"Deleting file {filepath} from database.")
+            rich_echo(f"Deleting file [filepath]{filepath}[/] from database.")
             count = exportdb.delete_data_for_filepath(filepath)
-            print(f"Deleted {count} {pluralize(count, 'record', 'records')}.")
+            rich_echo(
+                f"Deleted [num]{count}[/] {pluralize(count, 'record', 'records')}."
+            )
             sys.exit(0)
 
     if report:
@@ -363,27 +436,27 @@ def exportdb(
         report_filename = render_and_validate_report(report_template, "", export_dir)
         export_results = exportdb.get_export_results(run_id)
         if not export_results:
-            print(f"[red]No report results found for run ID {run_id}[/red]")
+            rich_echo(f"[error]No report results found for run ID {run_id}[/error]")
             sys.exit(1)
         try:
             report_writer = report_writer_factory(report_filename, append=append)
         except ValueError as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         report_writer.write(export_results)
         report_writer.close()
-        print(f"Wrote report to {report_filename}")
+        rich_echo(f"Wrote report to [filepath]{report_filename}[/]")
         sys.exit(0)
 
     if migrate:
         exportdb = ExportDB(export_db, export_dir)
         if upgraded := exportdb.was_upgraded:
-            print(
+            rich_echo(
-                f"Migrated export database {export_db} from version {upgraded[0]} to {upgraded[1]}"
+                f"Migrated export database [filepath]{export_db}[/] from version [num]{upgraded[0]}[/] to [num]{upgraded[1]}[/]"
             )
         else:
-            print(
+            rich_echo(
-                f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}"
+                f"Export database [filepath]{export_db}[/] is already at latest version [num]{OSXPHOTOS_EXPORTDB_VERSION}[/]"
             )
         sys.exit(0)
 
@@ -393,7 +466,7 @@ def exportdb(
             c = exportdb._conn.cursor()
             results = c.execute(sql)
         except Exception as e:
-            print(f"[red]Error: {e}[/red]")
+            rich_echo(f"[error]Error: {e}[/error]")
             sys.exit(1)
         else:
             for row in results:

osxphotos/export_db.py

@@ -1,10 +1,10 @@
 """ Helper class for managing database used by PhotoExporter for tracking state of exports and updates """
 
+from __future__ import annotations
 
 import datetime
 import gzip
 import json
-import logging
 import os
 import os.path
 import pathlib
@@ -32,7 +32,7 @@ __all__ = [
     "ExportDBTemp",
 ]
 
-OSXPHOTOS_EXPORTDB_VERSION = "7.1"
+OSXPHOTOS_EXPORTDB_VERSION = "8.0"
 OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
 
 # max retry attempts for methods which use tenacity.retry
@@ -532,6 +532,10 @@ class ExportDB:
             # add timestamp to export_data
             self._migrate_7_0_to_7_1(conn)
 
+        if version[1] < "8.0":
+            # add error to export_data
+            self._migrate_7_1_to_8_0(conn)
+
         conn.execute("VACUUM;")
         conn.commit()
 
@@ -739,6 +743,7 @@ class ExportDB:
         conn.commit()
 
     def _migrate_7_0_to_7_1(self, conn):
+        """Add timestamp column to export_data table and triggers to update it on insert and update."""
         c = conn.cursor()
         # timestamp column should not exist but this prevents error if migration is run on an already migrated database
         # reference #794
@@ -767,6 +772,16 @@ class ExportDB:
         )
         conn.commit()
 
+    def _migrate_7_1_to_8_0(self, conn):
+        """Add error column to export_data table"""
+        c = conn.cursor()
+        results = c.execute(
+            "SELECT COUNT(*) FROM pragma_table_info('export_data') WHERE name='error';"
+        ).fetchone()
+        if results[0] == 0:
+            c.execute("""ALTER TABLE export_data ADD COLUMN error JSON;""")
+        conn.commit()
+
     def _perform_db_maintenace(self, conn):
         """Perform database maintenance"""
         c = conn.cursor()
@@ -1133,6 +1148,35 @@ class ExportRecord:
             f"No timestamp found in database for {self._filepath_normalized}"
         )
 
+    @property
+    def error(self) -> dict[str, Any] | None:
+        """Return error value"""
+        conn = self._conn
+        c = conn.cursor()
+        if row := c.execute(
+            "SELECT error FROM export_data WHERE filepath_normalized = ?;",
+            (self._filepath_normalized,),
+        ).fetchone():
+            return json.loads(row[0]) if row[0] else None
+
+        raise ValueError(f"No error found in database for {self._filepath_normalized}")
+
+    @error.setter
+    def error(self, value: dict[str, Any] | None):
+        """Set error value"""
+        conn = self._conn
+        c = conn.cursor()
+        if value is None:
+            value = ""
+        # use default=str because some of the values are Path objects
+        error = json.dumps(value, default=str)
+        c.execute(
+            "UPDATE export_data SET error = ? WHERE filepath_normalized = ?;",
+            (error, self._filepath_normalized),
+        )
+        if not self._context_manager:
+            conn.commit()
+
     def asdict(self):
         """Return dict of self"""
         exifdata = json.loads(self.exifdata) if self.exifdata else None
@@ -1147,6 +1191,7 @@ class ExportRecord:
             "dest_sig": self.dest_sig,
             "export_options": self.export_options,
             "exifdata": exifdata,
+            "error": self.error,
             "photoinfo": photoinfo,
         }
 

osxphotos/export_db_utils.py

@@ -20,6 +20,7 @@ from .utils import noop
 
 __all__ = [
     "export_db_check_signatures",
+    "export_db_get_errors",
     "export_db_get_last_run",
     "export_db_get_version",
     "export_db_save_config_to_file",
@@ -40,10 +41,9 @@ def export_db_get_version(
     """returns version from export database as tuple of (osxphotos version, export_db version)"""
     conn = sqlite3.connect(str(dbfile))
     c = conn.cursor()
-    row = c.execute(
+    if row := c.execute(
         "SELECT osxphotos, exportdb FROM version ORDER BY id DESC LIMIT 1;"
-    ).fetchone()
+    ).fetchone():
-    if row:
         return (row[0], row[1])
     return (None, None)
 
@@ -80,11 +80,13 @@ def export_db_update_signatures(
         filepath = export_dir / filepath
         if not os.path.exists(filepath):
             skipped += 1
-            verbose_(f"[dark_orange]Skipping missing file[/dark_orange]: '{filepath}'")
+            verbose_(
+                f"[dark_orange]Skipping missing file[/dark_orange]: '[filepath]{filepath}[/]'"
+            )
             continue
         updated += 1
         file_sig = fileutil.file_sig(filepath)
-        verbose_(f"[green]Updating signature for[/green]: '{filepath}'")
+        verbose_(f"[green]Updating signature for[/green]: '[filepath]{filepath}[/]'")
         if not dry_run:
             c.execute(
                 "UPDATE export_data SET dest_mode = ?, dest_size = ?, dest_mtime = ? WHERE filepath_normalized = ?;",
@@ -103,14 +105,29 @@ def export_db_get_last_run(
     """Get last run from export database"""
     conn = sqlite3.connect(str(export_db))
     c = conn.cursor()
-    row = c.execute(
+    if row := c.execute(
         "SELECT datetime, args FROM runs ORDER BY id DESC LIMIT 1;"
-    ).fetchone()
+    ).fetchone():
-    if row:
         return row[0], row[1]
     return None, None
 
 
+def export_db_get_errors(
+    export_db: Union[str, pathlib.Path]
+) -> Tuple[Optional[str], Optional[str]]:
+    """Get errors from export database"""
+    conn = sqlite3.connect(str(export_db))
+    c = conn.cursor()
+    results = c.execute(
+        "SELECT filepath, uuid, timestamp, error FROM export_data WHERE error is not null ORDER BY timestamp DESC;"
+    ).fetchall()
+    results = [
+        f"[filepath]{row[0]}[/], [uuid]{row[1]}[/], [time]{row[2]}[/], [error]{row[3]}[/]"
+        for row in results
+    ]
+    return results
+
+
 def export_db_save_config_to_file(
     export_db: Union[str, pathlib.Path], config_file: Union[str, pathlib.Path]
 ) -> None:
@@ -138,9 +155,11 @@ def export_db_get_config(
     conn = sqlite3.connect(str(export_db))
     c = conn.cursor()
     row = c.execute("SELECT config FROM config ORDER BY id DESC LIMIT 1;").fetchone()
-    if not row:
+    return (
-        return ValueError("No config found in export_db")
+        config.load_from_str(row[0], override=override)
-    return config.load_from_str(row[0], override=override)
+        if row
+        else ValueError("No config found in export_db")
+    )
 
 
 def export_db_check_signatures(
@@ -168,16 +187,20 @@ def export_db_check_signatures(
         filepath = export_dir / filepath
         if not filepath.exists():
             skipped += 1
-            verbose_(f"[dark_orange]Skipping missing file[/dark_orange]: '{filepath}'")
+            verbose_(
+                f"[dark_orange]Skipping missing file[/dark_orange]: '[filepath]{filepath}[/]'"
+            )
             continue
         file_sig = fileutil.file_sig(filepath)
         file_rec = exportdb.get_file_record(filepath)
         if file_rec.dest_sig == file_sig:
             matched += 1
-            verbose_(f"[green]Signatures matched[/green]: '{filepath}'")
+            verbose_(f"[green]Signatures matched[/green]: '[filepath]{filepath}[/]'")
         else:
             notmatched += 1
-            verbose_(f"[deep_pink3]Signatures do not match[/deep_pink3]: '{filepath}'")
+            verbose_(
+                f"[deep_pink3]Signatures do not match[/deep_pink3]: '[filepath]{filepath}[/]'"
+            )
 
     return (matched, notmatched, skipped)
 
@@ -196,8 +219,7 @@ def export_db_touch_files(
 
     # open and close exportdb to ensure it gets migrated
     exportdb = ExportDB(dbfile, export_dir)
-    upgraded = exportdb.was_upgraded
+    if upgraded := exportdb.was_upgraded:
-    if upgraded:
         verbose_(
             f"Upgraded export database {dbfile} from version {upgraded[0]} to {upgraded[1]}"
         )
@@ -205,9 +227,9 @@ def export_db_touch_files(
 
     conn = sqlite3.connect(str(dbfile))
     c = conn.cursor()
-    # get most recent config
+    if row := c.execute(
-    row = c.execute("SELECT config FROM config ORDER BY id DESC LIMIT 1;").fetchone()
+        "SELECT config FROM config ORDER BY id DESC LIMIT 1;"
-    if row:
+    ).fetchone():
         config = toml.loads(row[0])
         try:
             photos_db_path = config["export"].get("db", None)
@@ -237,7 +259,7 @@ def export_db_touch_files(
         if not filepath.exists():
             skipped += 1
             verbose_(
-                f"[dark_orange]Skipping missing file (not in export directory)[/dark_orange]: '{filepath}'"
+                f"[dark_orange]Skipping missing file (not in export directory)[/dark_orange]: '[filepath]{filepath}[/]'"
             )
             continue
 
@@ -245,7 +267,7 @@ def export_db_touch_files(
         if not photo:
             skipped += 1
             verbose_(
-                f"[dark_orange]Skipping missing photo (did not find in Photos Library)[/dark_orange]: '{filepath}' ({uuid})"
+                f"[dark_orange]Skipping missing photo (did not find in Photos Library)[/dark_orange]: '[filepath]{filepath}[/]' ([uuid]{uuid}[/])"
             )
             continue
 
@@ -255,14 +277,14 @@ def export_db_touch_files(
         if mtime == ts:
             not_touched += 1
             verbose_(
-                f"[green]Skipping file (timestamp matches)[/green]: '{filepath}' [dodger_blue1]{isotime_from_ts(ts)} ({ts})[/dodger_blue1]"
+                f"[green]Skipping file (timestamp matches)[/green]: '[filepath]{filepath}[/]' [time]{isotime_from_ts(ts)} ({ts})[/time]"
             )
             continue
 
         touched += 1
         verbose_(
-            f"[deep_pink3]Touching file[/deep_pink3]: '{filepath}' "
+            f"[deep_pink3]Touching file[/deep_pink3]: '[filepath]{filepath}[/]' "
-            f"[dodger_blue1]{isotime_from_ts(mtime)} ({mtime}) -> {isotime_from_ts(ts)} ({ts})[/dodger_blue1]"
+            f"[time]{isotime_from_ts(mtime)} ({mtime}) -> {isotime_from_ts(ts)} ({ts})[/time]"
         )
 
         if not dry_run:

osxphotos/photoexporter.py

@@ -76,6 +76,7 @@ class ShouldUpdate(Enum):
     EXIFTOOL_DIFFERENT = 6
     EDITED_SIG_DIFFERENT = 7
     DIGEST_DIFFERENT = 8
+    UPDATE_ERRORS = 9
 
 
 class ExportError(Exception):
@@ -130,6 +131,7 @@ class ExportOptions:
         timeout (int, default=120): timeout in seconds used with use_photos_export
         touch_file (bool, default=False): if True, sets file's modification time upon photo date
         update (bool, default=False): if True export will run in update mode, that is, it will not export the photo if the current version already exists in the destination
+        update_errors (bool, default=False): if True photos that previously produced a warning or error will be re-exported; otherwise they will note be
         use_albums_as_keywords (bool, default = False): if True, will include album names in keywords when exporting metadata with exiftool or sidecar
         use_persons_as_keywords (bool, default = False): if True, will include person names in keywords when exporting metadata with exiftool or sidecar
         use_photos_export (bool, default=False): if True will attempt to export photo via applescript interaction with Photos even if not missing (see also use_photokit, download_missing)
@@ -176,6 +178,7 @@ class ExportOptions:
     timeout: int = 120
     touch_file: bool = False
     update: bool = False
+    update_errors: bool = False
     use_albums_as_keywords: bool = False
     use_persons_as_keywords: bool = False
     use_photokit: bool = False
@@ -701,6 +704,10 @@ class PhotoExporter:
         self, src: pathlib.Path, dest: pathlib.Path, options: ExportOptions
     ) -> t.Literal[True, False]:
         """Return True if photo should be updated, else False"""
+
+        # NOTE: The order of certain checks is important
+        # read the comments below to understand why before changing
+        
         export_db = options.export_db
         fileutil = options.fileutil
 
@@ -731,6 +738,14 @@ class PhotoExporter:
             # as it'll be None and bit_flags will be an int
             return ShouldUpdate.EXPORT_OPTIONS_DIFFERENT
 
+        if options.update_errors and file_record.error is not None:
+            # files that were exported but generated an error
+            # won't be updated unless --update-errors is specified
+            # for example, an exiftool error due to bad metadata
+            # that the user subsequently fixed should be updated; see #872
+            # this must be checked before exiftool which will return False if exif data matches
+            return ShouldUpdate.UPDATE_ERRORS
+
         if options.exiftool:
             current_exifdata = self.exiftool_json_sidecar(options=options)
             rv = current_exifdata != file_record.exifdata
@@ -1179,8 +1194,7 @@ class PhotoExporter:
 
         # set data in the database
         with export_db.create_or_get_file_record(dest_str, self.photo.uuid) as rec:
-            photoinfo = self.photo.json()
+            rec.photoinfo = self.photo.json()
-            rec.photoinfo = photoinfo
             rec.export_options = options.bit_flags
             # don't set src_sig as that is set above before any modifications by convert_to_jpeg or exiftool
             if not options.ignore_signature:
@@ -1190,6 +1204,17 @@ class PhotoExporter:
             if self.photo.hexdigest != rec.digest:
                 results.metadata_changed = [dest_str]
             rec.digest = self.photo.hexdigest
+            # save errors to the export database (#872)
+            if (
+                results.error
+                or exif_results.exiftool_error
+                or exif_results.exiftool_warning
+            ):
+                rec.error = {
+                    "error": results.error,
+                    "exiftool_error": exif_results.exiftool_error,
+                    "exiftool_warning": exif_results.exiftool_warning,
+                }
 
         return results
 

tests/test_cli.py

@@ -2292,24 +2292,72 @@ def test_export_exiftool_favorite_rating():
     cwd = os.getcwd()
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
-        for uuid in CLI_EXIFTOOL:
+        result = runner.invoke(
-            result = runner.invoke(
+            export,
-                export,
+            [
-                [
+                os.path.join(cwd, PHOTOS_DB_15_7),
-                    os.path.join(cwd, PHOTOS_DB_15_7),
+                ".",
-                    ".",
+                "-V",
-                    "-V",
+                "--exiftool",
-                    "--exiftool",
+                "--uuid",
-                    "--uuid",
+                UUID_FAVORITE,
-                    UUID_FAVORITE,
+                "--uuid",
-                    "--uuid",
+                UUID_NOT_FAVORITE,
-                    UUID_NOT_FAVORITE,
+                "--favorite-rating",
-                    "--favorite-rating",
+            ],
-                ],
+        )
-            )
+        assert result.exit_code == 0
-            assert result.exit_code == 0
+        assert ExifTool(FILE_FAVORITE).asdict()["XMP:Rating"] == 5
-            assert ExifTool(FILE_FAVORITE).asdict()["XMP:Rating"] == 5
+        assert ExifTool(FILE_NOT_FAVORITE).asdict()["XMP:Rating"] == 0
-            assert ExifTool(FILE_NOT_FAVORITE).asdict()["XMP:Rating"] == 0
+
+
+@pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
+def test_export_exiftool_update_errors():
+    """Test export with --update-errors, #872"""
+    runner = CliRunner()
+    cwd = os.getcwd()
+
+    # first, normal export with --exiftool
+    # some of the files will have errors / warnings from exiftool
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                ".",
+                "-V",
+                "--exiftool",
+            ],
+        )
+
+        # now run update; none of files should be updated
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                ".",
+                "-V",
+                "--exiftool",
+                "--update",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "updated: 0" in result.output
+
+        # now run update-errors; only files with errors should be updated
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                ".",
+                "-V",
+                "--exiftool",
+                "--update",
+                "--update-errors",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "updated: 4" in result.output
 
 
 def test_export_edited_suffix():