Feature post function return 1136 (#1147)

* Changed return signature for post_function

* Updated post_function.py example

* Added tests for post_function returns ExportResults

examples/post_function.py

@@ -1,57 +1,88 @@
 """ Example function for use with osxphotos export --post-function option """
 
-from typing import Callable
+from __future__ import annotations
+
+import pathlib
+from typing import Any, Callable
 
 from osxphotos import ExportResults, PhotoInfo
+from osxphotos.cli import echo_error
 
 
 def post_function(
-    photo: PhotoInfo, results: ExportResults, verbose: Callable, **kwargs
+    photo: PhotoInfo, results: ExportResults, verbose: Callable[[Any], None], **kwargs
-):
+) -> ExportResults | None:
     """Call this with osxphotos export /path/to/export --post-function post_function.py::post_function
         This will get called immediately after the photo has been exported
 
     Args:
         photo: PhotoInfo instance for the photo that's just been exported
+            see https://rhettbull.github.io/osxphotos/reference.html#osxphotos.PhotoInfo for details
         results: ExportResults instance with information about the files associated with the exported photo
+            see https://rhettbull.github.io/osxphotos/reference.html#osxphotos.ExportResults for details
         verbose: A function to print verbose output if --verbose is set; if --verbose is not set, acts as a no-op (nothing gets printed)
         **kwargs: reserved for future use; recommend you include **kwargs so your function still works if additional arguments are added in future versions
 
+    Returns:
+        ExportResults instance or None
+        If returning an ExportResults instance, it must be a new instance; do not modify the instance passed in as an argument.
+        You should set only the following values in ExportResults:
+            user_written: list[str], list of files written by your function
+            user_skipped: list[str], list of files skipped by your function
+            user_error: list[tuple[str, str]], list of tuples of (filename, error) for any errors generated by your function
+        For full description of ExportResults see: https://rhettbull.github.io/osxphotos/reference.html#osxphotos.ExportResults
+
     Notes:
         Use verbose(str) instead of print if you want your function to conditionally output text depending on --verbose flag
         Any string printed with verbose that contains "warning" or "error" (case-insensitive) will be printed with the appropriate warning or error color
-        Will not be called if --dry-run flag is enabled
+        The function will not be called if --dry-run flag is enabled
-        Will be called immediately after export and before any --post-command commands are executed
+        The function be called immediately after export and before any --post-command commands are executed
+        If your function does not write any files, you can optionally return None instead of an ExportResults instance
+        If you return ExportResults, any files in user_written or user_skipped will be included when
+            writing reports and will be preserved when using --cleanup
+        If you want files previously written by the post_function, but not written this time,
+            to be preserved when using --cleanup, you should include them in user_skipped
+
+        If the function raises an exception, osxphotos will abort the export and exit with an error.
+        If you want the export to continue, you should catch any exceptions and return an ExportResults instance
+            with the error(s) in specified in ExportResults.user_error which is a list of tuples of (filename, error)
+
+        The verbose function can be used to print text to stdout if --verbose is set
+            If --verbose is not set, verbose acts as a no-op (nothing gets printed)
+            Verbose output may be stylized with tags as follows; tags must be enclosed in square brackets
+            and closed with [/] to end the style:
+                [change]: something change
+                [no_change]: indicate no change
+                [count]: a count
+                [error]: an error
+                [filename]: a filename
+                [filepath]: a filepath
+                [num]: a number
+                [time]: a time or date
+                [tz]: a timezone
+                [warning]: a warning
+                [uuid]: a uuid
     """
 
-    # ExportResults has the following properties
+    post_results = ExportResults()
-    # fields with filenames contain the full path to the file
+    for filename in results.exported + results.skipped:
-    # exported: list of all files exported
-    # new: list of all new files exported (--update)
-    # updated: list of all files updated (--update)
-    # skipped: list of all files skipped (--update)
-    # exif_updated: list of all files that were updated with --exiftool
-    # touched: list of all files that had date updated with --touch-file
-    # converted_to_jpeg: list of files converted to jpeg with --convert-to-jpeg
-    # sidecar_json_written: list of all JSON sidecar files written
-    # sidecar_json_skipped: list of all JSON sidecar files skipped (--update)
-    # sidecar_exiftool_written: list of all exiftool sidecar files written
-    # sidecar_exiftool_skipped: list of all exiftool sidecar files skipped (--update)
-    # sidecar_xmp_written: list of all XMP sidecar files written
-    # sidecar_xmp_skipped: list of all XMP sidecar files skipped (--update)
-    # missing: list of all missing files
-    # error: list tuples of (filename, error) for any errors generated during export
-    # exiftool_warning: list of tuples of (filename, warning) for any warnings generated by exiftool with --exiftool
-    # exiftool_error: list of tuples of (filename, error) for any errors generated by exiftool with --exiftool
-    # xattr_written: list of files that had extended attributes written
-    # xattr_skipped: list of files that where extended attributes were skipped (--update)
-    # deleted_files: list of deleted files
-    # deleted_directories: list of deleted directories
-    # exported_album: list of tuples of (filename, album_name) for exported files added to album with --add-exported-to-album
-    # skipped_album: list of tuples of (filename, album_name) for skipped files added to album with --add-skipped-to-album
-    # missing_album: list of tuples of (filename, album_name) for missing files added to album with --add-missing-to-album
-    # metadata_changed: list of filenames that had metadata changes since last export
-
-    for filename in results.exported:
         # do your processing here
-        verbose(f"post_function: {photo.original_filename} exported as {filename}")
+
+        # simulate doing some processing
+        new_filename = pathlib.Path(f"{filename}.new")
+        if new_filename.exists():
+            verbose(f"Skipping file [filepath]{new_filename}[/] as it already exists")
+            post_results.user_skipped.append(new_filename)
+        else:
+            verbose(f"Writing new file [filepath]{new_filename}[/]")
+            new_filename.touch()
+            post_results.user_written.append(new_filename)
+
+        # if you encounter an error, add it to user_error
+        # echo_error will print the error to stderr with the appropriate formatting
+
+        # echo_error(f"Encountered an error processing [filepath]{filename}[/]: [error]some error[/]")
+        # post_results.user_error.append((filename, "some error"))
+
+    # if your function does not write any files, you can return None
+    return post_results

osxphotos/cli/export.py

@@ -1570,17 +1570,14 @@ def export(
                     )
 
                 # run post functions
-                if post_function:
+                if run_results := run_post_function(
-                    for function in post_function:
+                    photo=p,
-                        # post function is tuple of (function, filename.py::function_name)
+                    post_function=post_function,
-                        verbose(f"Calling post-function [bold]{function[1]}")
+                    export_results=export_results,
-                        if not dry_run:
+                    verbose=verbose,
-                            try:
+                    dry_run=dry_run,
-                                function[0](p, export_results, verbose)
+                ):
-                            except Exception as e:
+                    export_results += run_results
-                                rich_echo_error(
-                                    f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
-                                )
 
                 # run post command
                 run_post_command(
@@ -1768,6 +1765,8 @@ def export(
             + results.sidecar_xmp_skipped
             + results.sidecar_user_written
             + results.sidecar_user_skipped
+            + results.user_written
+            + results.user_skipped
             # include missing so a file that was already in export directory
             # but was missing on --update doesn't get deleted
             # (better to have old version than none)
@@ -2855,6 +2854,39 @@ def write_extended_attributes(
     return list(written), [f for f in skipped if f not in written]
 
 
+def run_post_function(
+    photo: osxphotos.PhotoInfo,
+    post_function: tuple[
+        tuple[
+            Callable[
+                [osxphotos.PhotoInfo, ExportResults, Callable[[Any], None]],
+                None | ExportResults,
+            ],
+            str,
+        ],
+        ...,
+    ],
+    export_results: ExportResults,
+    verbose: Callable[[Any], None],
+    dry_run: bool,
+) -> ExportResults:
+    """Run the --post-function functions"""
+    returned_results = ExportResults()
+    for function in post_function:
+        # post function is tuple of (function, filename.py::function_name)
+        verbose(f"Calling post-function [bold]{function[1]}")
+        if not dry_run:
+            try:
+                if results := function[0](photo, export_results, verbose):
+                    returned_results += results
+            except Exception as e:
+                rich_echo_error(
+                    f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
+                )
+                raise e
+    return returned_results
+
+
 def run_post_command(
     photo: osxphotos.PhotoInfo,
     post_command: tuple[tuple[str, str]],

osxphotos/cli/report_writer.py

@@ -99,6 +99,10 @@ class ExportReportWriterCSV(ReportWriterABC):
             "cleanup_deleted_directory",
             "exported_album",
             "sidecar_user",
+            "sidecar_user_error",
+            "user_written",
+            "user_skipped",
+            "user_error",
         ]
 
         mode = "a" if append else "w"
@@ -198,9 +202,9 @@ class ExportReportWriterSQLite(ReportWriterABC):
             cursor = self._conn.cursor()
             cursor.execute(
                 "INSERT INTO report "
-                "(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id, sidecar_user) "  # noqa
+                "(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id, sidecar_user, sidecar_user_error, user_written, user_skipped, user_error) "  # noqa
                 "VALUES "
-                "(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id, :sidecar_user);",  # noqa
+                "(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id, :sidecar_user, :sidecar_user_error, :user_written, :user_skipped, :user_error);",  # noqa
                 data,
             )
         self._conn.commit()
@@ -270,6 +274,32 @@ class ExportReportWriterSQLite(ReportWriterABC):
             )
             self._conn.commit()
 
+        # migrate report table and add sidecar_user_error if needed (#1123)
+        if "sidecar_user_error" not in sqlite_columns(self._conn, "report"):
+            self._conn.cursor().execute(
+                "ALTER TABLE report ADD COLUMN sidecar_user_error TEXT;"
+            )
+            self._conn.commit()
+
+        # migrate report table and add user_written, skipped, error if needed (#1136)
+        if "user_written" not in sqlite_columns(self._conn, "report"):
+            self._conn.cursor().execute(
+                "ALTER TABLE report ADD COLUMN user_written INTEGER;"
+            )
+            self._conn.commit()
+
+        if "user_skipped" not in sqlite_columns(self._conn, "report"):
+            self._conn.cursor().execute(
+                "ALTER TABLE report ADD COLUMN user_skipped INTEGER;"
+            )
+            self._conn.commit()
+
+        if "user_error" not in sqlite_columns(self._conn, "report"):
+            self._conn.cursor().execute(
+                "ALTER TABLE report ADD COLUMN user_error TEXT;"
+            )
+            self._conn.commit()
+
         # create report_summary view
         c.execute(
             """
@@ -356,6 +386,10 @@ def prepare_export_results_for_writing(
                 "cleanup_deleted_directory": false,
                 "exported_album": "",
                 "sidecar_user": false,
+                "sidecar_user_error": "",
+                "user_written": false,
+                "user_skipped": false,
+                "user_error": "",
             }
 
     for result in export_results.exported:
@@ -438,6 +472,20 @@ def prepare_export_results_for_writing(
         all_results[str(result)]["sidecar_user"] = true
         all_results[str(result)]["skipped"] = true
 
+    for result in export_results.sidecar_user_error:
+        all_results[str(result[0])]["sidecar_user_error"] = result[1]
+
+    for result in export_results.user_written:
+        all_results[str(result)]["user_written"] = true
+        all_results[str(result)]["exported"] = true
+
+    for result in export_results.user_skipped:
+        all_results[str(result)]["user_skipped"] = true
+        all_results[str(result)]["skipped"] = true
+
+    for result in export_results.user_error:
+        all_results[str(result[0])]["user_error"] = result[1]
+
     return all_results
 
 

osxphotos/photoexporter.py

@@ -300,41 +300,90 @@ class ExportResults:
         "updated",
         "xattr_skipped",
         "xattr_written",
+        "user_written",
+        "user_skipped",
+        "user_error",
     ]
 
     def __init__(
         self,
-        converted_to_jpeg=None,
+        converted_to_jpeg: list[str] | None = None,
-        deleted_directories=None,
+        deleted_directories: list[str] | None = None,
-        deleted_files=None,
+        deleted_files: list[str] | None = None,
-        error=None,
+        error: list[str] | None = None,
-        exif_updated=None,
+        exif_updated: list[str] | None = None,
-        exiftool_error=None,
+        exiftool_error: list[tuple[str, str]] | None = None,
-        exiftool_warning=None,
+        exiftool_warning: list[tuple[str, str]] | None = None,
-        exported=None,
+        exported: list[str] | None = None,
-        exported_album=None,
+        exported_album: list[tuple[str, str]] | None = None,
-        metadata_changed=None,
+        metadata_changed: list[str] | None = None,
-        missing=None,
+        missing: list[str] | None = None,
-        missing_album=None,
+        missing_album: list[tuple[str, str]] | None = None,
-        new=None,
+        new: list[str] | None = None,
-        aae_written=None,
+        aae_written: list[str] | None = None,
-        sidecar_exiftool_skipped=None,
+        sidecar_exiftool_skipped: list[str] | None = None,
-        sidecar_exiftool_written=None,
+        sidecar_exiftool_written: list[str] | None = None,
-        sidecar_json_skipped=None,
+        sidecar_json_skipped: list[str] | None = None,
-        sidecar_json_written=None,
+        sidecar_json_written: list[str] | None = None,
-        sidecar_xmp_skipped=None,
+        sidecar_xmp_skipped: list[str] | None = None,
-        sidecar_xmp_written=None,
+        sidecar_xmp_written: list[str] | None = None,
-        sidecar_user_written=None,
+        sidecar_user_written: list[str] | None = None,
-        sidecar_user_skipped=None,
+        sidecar_user_skipped: list[str] | None = None,
-        sidecar_user_error=None,
+        sidecar_user_error: list[tuple[str, str]] | None = None,
-        skipped=None,
+        skipped: list[str] | None = None,
-        skipped_album=None,
+        skipped_album: list[tuple[str, str]] | None = None,
-        to_touch=None,
+        to_touch: list[str] | None = None,
-        touched=None,
+        touched: list[str] | None = None,
-        updated=None,
+        updated: list[str] | None = None,
-        xattr_skipped=None,
+        xattr_skipped: list[str] | None = None,
-        xattr_written=None,
+        xattr_written: list[str] | None = None,
+        user_written: list[str] | None = None,
+        user_skipped: list[str] | None = None,
+        user_error: list[tuple[str, str]] | None = None,
     ):
+        """ExportResults data class to hold results of export.
+
+        Args:
+            converted_to_jpeg: list of files converted to jpeg
+            deleted_directories: list of directories deleted
+            deleted_files: list of files deleted
+            error: list of tuples of (filename, error) for any errors generated during export
+            exif_updated: list of files where exif data was updated with exiftool
+            exiftool_error: list of tuples of (filename, error) for any errors generated by exiftool
+            exiftool_warning: list of tuples of (filename, warning) for any warnings generated by exiftool
+            exported: list of files exported
+            exported_album: list of tuples of (file, album) for any files exported to an album
+            metadata_changed: list of filenames that had metadata changes since last export
+            missing: list of files that were missing
+            missing_album: list of tuples of (file, album) for any files that were missing from an album
+            new: list of files that were new
+            aae_written: list of files where .AAE file was written
+            sidecar_exiftool_skipped: list of files where exiftool sidecar was skipped
+            sidecar_exiftool_written: list of files where exiftool sidecar was written
+            sidecar_json_skipped: list of files where json sidecar was skipped
+            sidecar_json_written: list of files where json sidecar was written
+            sidecar_xmp_skipped: list of files where xmp sidecar was skipped
+            sidecar_xmp_written: list of files where xmp sidecar was written
+            sidecar_user_written: list of files where user sidecar was written
+            sidecar_user_skipped: list of files where user sidecar was skipped
+            sidecar_user_error: list of tuples of (filename, error) for any errors generated by user sidecar
+            skipped: list of files that were skipped
+            skipped_album: list of tuples of (file, album) for any files that were skipped from an album
+            to_touch: list of files that were touched
+            touched: list of files that were touched
+            updated: list of files that were updated
+            xattr_skipped: list of files where xattr was skipped
+            xattr_written: list of files where xattr was written
+            user_written: list of files written by user post_function
+            user_skipped: list of files skipped by user post_function
+            user_error: list of tuples of (filename, error) for any errors generated by user post_function
+
+        Notes:
+            Each attribute is a list of files or None if no files for that attribute.
+            Error and warning attributes are a list of tuples of (filename, error) where filename is the file that caused the error and error is the error message.
+            Album attributes are a list of tuples of (file, album) where file is the file exported and album is the album it was exported to.
+            ExportResults can be added together with the += operator to combine results as the export progresses.
+        """
         local_vars = locals()
         self._datetime = datetime.now().isoformat()
         for attr in self.attributes:
@@ -370,11 +419,14 @@ class ExportResults:
             + self.sidecar_user_written
             + self.sidecar_user_skipped
             + self.missing
+            + self.user_written
+            + self.user_skipped
         )
         files += [x[0] for x in self.exiftool_warning]
         files += [x[0] for x in self.exiftool_error]
         files += [x[0] for x in self.error]
         files += [x[0] for x in self.sidecar_user_error]
+        files += [x[0] for x in self.user_error]
 
         return list(set(files))
 

tests/test_cli.py

@@ -556,7 +556,7 @@ CLI_EXPORTED_FILENAME_TEMPLATE_LONG_DESCRIPTION = [
     "pellentesque eu, pretium q.tif"
 ]
 
-CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B"
+CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B"  # Pumkins2.jpg
 CLI_EXPORT_UUID_STATUE = "3DD2C897-F19E-4CA6-8C22-B027D5A71907"
 CLI_EXPORT_UUID_KEYWORD_PATHSEP = "7783E8E6-9CAC-40F3-BE22-81FB7051C266"
 CLI_EXPORT_UUID_LONG_DESCRIPTION = "8846E3E6-8AC8-4857-8448-E3D025784410"
@@ -8823,7 +8823,7 @@ def test_export_post_function_exception():
                 "-V",
             ],
         )
-        assert result.exit_code == 0
+        assert result.exit_code != 0
         assert "Error running post-function" in result.output
 
 
@@ -8862,6 +8862,57 @@ def test_export_post_function_bad_value():
         assert "Could not load function" in result.output
 
 
+def test_export_post_function_results():
+    """Test --post-function with returned ExportResults, uses the post_function in examples/post_function.py"""
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        tempdir = os.getcwd()
+        result = runner.invoke(
+            cli_main,
+            [
+                "export",
+                ".",
+                "--db",
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                "--name",
+                "wedding",
+                "-V",
+                "--post-function",
+                f"{cwd}/examples/post_function.py::post_function",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Writing new file" in result.output
+        export_files = glob.glob(os.path.join(f"{tempdir}", "*"))
+        assert os.path.join(tempdir, "wedding.jpg.new") in export_files
+
+        # run again with --update and --cleanup
+        result = runner.invoke(
+            cli_main,
+            [
+                "export",
+                ".",
+                "--db",
+                os.path.join(cwd, PHOTOS_DB_15_7),
+                "--name",
+                "wedding",
+                "-V",
+                "--post-function",
+                f"{cwd}/examples/post_function.py::post_function",
+                "--update",
+                "--cleanup",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Skipping file" in result.output
+        assert "Deleted: 0 files" in result.output
+        export_files = glob.glob(os.path.join(f"{tempdir}", "*"))
+        assert os.path.join(tempdir, "wedding.jpg.new") in export_files
+
+
 def test_export_directory_template_function():
     """Test --directory with template function"""