forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature post function return 1136 (#1147)

Browse Source 
* Changed return signature for post_function

* Updated post_function.py example

* Added tests for post_function returns ExportResults
This commit is contained in:
Rhet Turnbull 2023-08-06 08:47:42 -07:00 committed by GitHub
parent 0ff2d50004
commit b833cde599
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 294 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
examples/post_function.py
View File

@ -1,57 +1,88 @@
""" Example function for use with osxphotos export --post-function option """
from typing import Callable
from __future__ import annotations
import pathlib
from typing import Any, Callable
from osxphotos import ExportResults, PhotoInfo
from osxphotos.cli import echo_error
def post_function(
photo: PhotoInfo, results: ExportResults, verbose: Callable, **kwargs
):
photo: PhotoInfo, results: ExportResults, verbose: Callable[[Any], None], **kwargs
) -> ExportResults | None:
"""Call this with osxphotos export /path/to/export --post-function post_function.py::post_function
This will get called immediately after the photo has been exported
Args:
photo: PhotoInfo instance for the photo that's just been exported
see https://rhettbull.github.io/osxphotos/reference.html#osxphotos.PhotoInfo for details
results: ExportResults instance with information about the files associated with the exported photo
see https://rhettbull.github.io/osxphotos/reference.html#osxphotos.ExportResults for details
verbose: A function to print verbose output if --verbose is set; if --verbose is not set, acts as a no-op (nothing gets printed)
**kwargs: reserved for future use; recommend you include **kwargs so your function still works if additional arguments are added in future versions
Returns:
ExportResults instance or None
If returning an ExportResults instance, it must be a new instance; do not modify the instance passed in as an argument.
You should set only the following values in ExportResults:
user_written: list[str], list of files written by your function
user_skipped: list[str], list of files skipped by your function
user_error: list[tuple[str, str]], list of tuples of (filename, error) for any errors generated by your function
For full description of ExportResults see: https://rhettbull.github.io/osxphotos/reference.html#osxphotos.ExportResults
Notes:
Use verbose(str) instead of print if you want your function to conditionally output text depending on --verbose flag
Any string printed with verbose that contains "warning" or "error" (case-insensitive) will be printed with the appropriate warning or error color
Will not be called if --dry-run flag is enabled
Will be called immediately after export and before any --post-command commands are executed
The function will not be called if --dry-run flag is enabled
The function be called immediately after export and before any --post-command commands are executed
If your function does not write any files, you can optionally return None instead of an ExportResults instance
If you return ExportResults, any files in user_written or user_skipped will be included when
writing reports and will be preserved when using --cleanup
If you want files previously written by the post_function, but not written this time,
to be preserved when using --cleanup, you should include them in user_skipped
If the function raises an exception, osxphotos will abort the export and exit with an error.
If you want the export to continue, you should catch any exceptions and return an ExportResults instance
with the error(s) in specified in ExportResults.user_error which is a list of tuples of (filename, error)
The verbose function can be used to print text to stdout if --verbose is set
If --verbose is not set, verbose acts as a no-op (nothing gets printed)
Verbose output may be stylized with tags as follows; tags must be enclosed in square brackets
and closed with [/] to end the style:
[change]: something change
[no_change]: indicate no change
[count]: a count
[error]: an error
[filename]: a filename
[filepath]: a filepath
[num]: a number
[time]: a time or date
[tz]: a timezone
[warning]: a warning
[uuid]: a uuid
"""
# ExportResults has the following properties
# fields with filenames contain the full path to the file
# exported: list of all files exported
# new: list of all new files exported (--update)
# updated: list of all files updated (--update)
# skipped: list of all files skipped (--update)
# exif_updated: list of all files that were updated with --exiftool
# touched: list of all files that had date updated with --touch-file
# converted_to_jpeg: list of files converted to jpeg with --convert-to-jpeg
# sidecar_json_written: list of all JSON sidecar files written
# sidecar_json_skipped: list of all JSON sidecar files skipped (--update)
# sidecar_exiftool_written: list of all exiftool sidecar files written
# sidecar_exiftool_skipped: list of all exiftool sidecar files skipped (--update)
# sidecar_xmp_written: list of all XMP sidecar files written
# sidecar_xmp_skipped: list of all XMP sidecar files skipped (--update)
# missing: list of all missing files
# error: list tuples of (filename, error) for any errors generated during export
# exiftool_warning: list of tuples of (filename, warning) for any warnings generated by exiftool with --exiftool
# exiftool_error: list of tuples of (filename, error) for any errors generated by exiftool with --exiftool
# xattr_written: list of files that had extended attributes written
# xattr_skipped: list of files that where extended attributes were skipped (--update)
# deleted_files: list of deleted files
# deleted_directories: list of deleted directories
# exported_album: list of tuples of (filename, album_name) for exported files added to album with --add-exported-to-album
# skipped_album: list of tuples of (filename, album_name) for skipped files added to album with --add-skipped-to-album
# missing_album: list of tuples of (filename, album_name) for missing files added to album with --add-missing-to-album
# metadata_changed: list of filenames that had metadata changes since last export
for filename in results.exported:
post_results = ExportResults()
for filename in results.exported + results.skipped:
# do your processing here
verbose(f"post_function: {photo.original_filename} exported as {filename}")
# simulate doing some processing
new_filename = pathlib.Path(f"{filename}.new")
if new_filename.exists():
verbose(f"Skipping file [filepath]{new_filename}[/] as it already exists")
post_results.user_skipped.append(new_filename)
else:
verbose(f"Writing new file [filepath]{new_filename}[/]")
new_filename.touch()
post_results.user_written.append(new_filename)
# if you encounter an error, add it to user_error
# echo_error will print the error to stderr with the appropriate formatting
# echo_error(f"Encountered an error processing [filepath]{filename}[/]: [error]some error[/]")
# post_results.user_error.append((filename, "some error"))
# if your function does not write any files, you can return None
return post_results

54
osxphotos/cli/export.py
View File

@ -1570,17 +1570,14 @@ def export(
)
# run post functions
if post_function:
for function in post_function:
# post function is tuple of (function, filename.py::function_name)
verbose(f"Calling post-function [bold]{function[1]}")
if not dry_run:
try:
function[0](p, export_results, verbose)
except Exception as e:
rich_echo_error(
f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
)
if run_results := run_post_function(
photo=p,
post_function=post_function,
export_results=export_results,
verbose=verbose,
dry_run=dry_run,
):
export_results += run_results
# run post command
run_post_command(
@ -1768,6 +1765,8 @@ def export(
+ results.sidecar_xmp_skipped
+ results.sidecar_user_written
+ results.sidecar_user_skipped
+ results.user_written
+ results.user_skipped
# include missing so a file that was already in export directory
# but was missing on --update doesn't get deleted
# (better to have old version than none)
@ -2855,6 +2854,39 @@ def write_extended_attributes(
return list(written), [f for f in skipped if f not in written]
def run_post_function(
photo: osxphotos.PhotoInfo,
post_function: tuple[
tuple[
Callable[
[osxphotos.PhotoInfo, ExportResults, Callable[[Any], None]],
None | ExportResults,
],
str,
],
...,
],
export_results: ExportResults,
verbose: Callable[[Any], None],
dry_run: bool,
) -> ExportResults:
"""Run the --post-function functions"""
returned_results = ExportResults()
for function in post_function:
# post function is tuple of (function, filename.py::function_name)
verbose(f"Calling post-function [bold]{function[1]}")
if not dry_run:
try:
if results := function[0](photo, export_results, verbose):
returned_results += results
except Exception as e:
rich_echo_error(
f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
)
raise e
return returned_results
def run_post_command(
photo: osxphotos.PhotoInfo,
post_command: tuple[tuple[str, str]],

52
osxphotos/cli/report_writer.py
View File

@ -99,6 +99,10 @@ class ExportReportWriterCSV(ReportWriterABC):
"cleanup_deleted_directory",
"exported_album",
"sidecar_user",
"sidecar_user_error",
"user_written",
"user_skipped",
"user_error",
]
mode = "a" if append else "w"
@ -198,9 +202,9 @@ class ExportReportWriterSQLite(ReportWriterABC):
cursor = self._conn.cursor()
cursor.execute(
"INSERT INTO report "
"(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id, sidecar_user) " # noqa
"(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id, sidecar_user, sidecar_user_error, user_written, user_skipped, user_error) " # noqa
"VALUES "
"(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id, :sidecar_user);", # noqa
"(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id, :sidecar_user, :sidecar_user_error, :user_written, :user_skipped, :user_error);", # noqa
data,
)
self._conn.commit()
@ -270,6 +274,32 @@ class ExportReportWriterSQLite(ReportWriterABC):
)
self._conn.commit()
# migrate report table and add sidecar_user_error if needed (#1123)
if "sidecar_user_error" not in sqlite_columns(self._conn, "report"):
self._conn.cursor().execute(
"ALTER TABLE report ADD COLUMN sidecar_user_error TEXT;"
)
self._conn.commit()
# migrate report table and add user_written, skipped, error if needed (#1136)
if "user_written" not in sqlite_columns(self._conn, "report"):
self._conn.cursor().execute(
"ALTER TABLE report ADD COLUMN user_written INTEGER;"
)
self._conn.commit()
if "user_skipped" not in sqlite_columns(self._conn, "report"):
self._conn.cursor().execute(
"ALTER TABLE report ADD COLUMN user_skipped INTEGER;"
)
self._conn.commit()
if "user_error" not in sqlite_columns(self._conn, "report"):
self._conn.cursor().execute(
"ALTER TABLE report ADD COLUMN user_error TEXT;"
)
self._conn.commit()
# create report_summary view
c.execute(
"""
@ -356,6 +386,10 @@ def prepare_export_results_for_writing(
"cleanup_deleted_directory": false,
"exported_album": "",
"sidecar_user": false,
"sidecar_user_error": "",
"user_written": false,
"user_skipped": false,
"user_error": "",
}
for result in export_results.exported:
@ -438,6 +472,20 @@ def prepare_export_results_for_writing(
all_results[str(result)]["sidecar_user"] = true
all_results[str(result)]["skipped"] = true
for result in export_results.sidecar_user_error:
all_results[str(result[0])]["sidecar_user_error"] = result[1]
for result in export_results.user_written:
all_results[str(result)]["user_written"] = true
all_results[str(result)]["exported"] = true
for result in export_results.user_skipped:
all_results[str(result)]["user_skipped"] = true
all_results[str(result)]["skipped"] = true
for result in export_results.user_error:
all_results[str(result[0])]["user_error"] = result[1]
return all_results

112
osxphotos/photoexporter.py
View File

@ -300,41 +300,90 @@ class ExportResults:
"updated",
"xattr_skipped",
"xattr_written",
"user_written",
"user_skipped",
"user_error",
]
def __init__(
self,
converted_to_jpeg=None,
deleted_directories=None,
deleted_files=None,
error=None,
exif_updated=None,
exiftool_error=None,
exiftool_warning=None,
exported=None,
exported_album=None,
metadata_changed=None,
missing=None,
missing_album=None,
new=None,
aae_written=None,
sidecar_exiftool_skipped=None,
sidecar_exiftool_written=None,
sidecar_json_skipped=None,
sidecar_json_written=None,
sidecar_xmp_skipped=None,
sidecar_xmp_written=None,
sidecar_user_written=None,
sidecar_user_skipped=None,
sidecar_user_error=None,
skipped=None,
skipped_album=None,
to_touch=None,
touched=None,
updated=None,
xattr_skipped=None,
xattr_written=None,
converted_to_jpeg: list[str] | None = None,
deleted_directories: list[str] | None = None,
deleted_files: list[str] | None = None,
error: list[str] | None = None,
exif_updated: list[str] | None = None,
exiftool_error: list[tuple[str, str]] | None = None,
exiftool_warning: list[tuple[str, str]] | None = None,
exported: list[str] | None = None,
exported_album: list[tuple[str, str]] | None = None,
metadata_changed: list[str] | None = None,
missing: list[str] | None = None,
missing_album: list[tuple[str, str]] | None = None,
new: list[str] | None = None,
aae_written: list[str] | None = None,
sidecar_exiftool_skipped: list[str] | None = None,
sidecar_exiftool_written: list[str] | None = None,
sidecar_json_skipped: list[str] | None = None,
sidecar_json_written: list[str] | None = None,
sidecar_xmp_skipped: list[str] | None = None,
sidecar_xmp_written: list[str] | None = None,
sidecar_user_written: list[str] | None = None,
sidecar_user_skipped: list[str] | None = None,
sidecar_user_error: list[tuple[str, str]] | None = None,
skipped: list[str] | None = None,
skipped_album: list[tuple[str, str]] | None = None,
to_touch: list[str] | None = None,
touched: list[str] | None = None,
updated: list[str] | None = None,
xattr_skipped: list[str] | None = None,
xattr_written: list[str] | None = None,
user_written: list[str] | None = None,
user_skipped: list[str] | None = None,
user_error: list[tuple[str, str]] | None = None,
):
"""ExportResults data class to hold results of export.
Args:
converted_to_jpeg: list of files converted to jpeg
deleted_directories: list of directories deleted
deleted_files: list of files deleted
error: list of tuples of (filename, error) for any errors generated during export
exif_updated: list of files where exif data was updated with exiftool
exiftool_error: list of tuples of (filename, error) for any errors generated by exiftool
exiftool_warning: list of tuples of (filename, warning) for any warnings generated by exiftool
exported: list of files exported
exported_album: list of tuples of (file, album) for any files exported to an album
metadata_changed: list of filenames that had metadata changes since last export
missing: list of files that were missing
missing_album: list of tuples of (file, album) for any files that were missing from an album
new: list of files that were new
aae_written: list of files where .AAE file was written
sidecar_exiftool_skipped: list of files where exiftool sidecar was skipped
sidecar_exiftool_written: list of files where exiftool sidecar was written
sidecar_json_skipped: list of files where json sidecar was skipped
sidecar_json_written: list of files where json sidecar was written
sidecar_xmp_skipped: list of files where xmp sidecar was skipped
sidecar_xmp_written: list of files where xmp sidecar was written
sidecar_user_written: list of files where user sidecar was written
sidecar_user_skipped: list of files where user sidecar was skipped
sidecar_user_error: list of tuples of (filename, error) for any errors generated by user sidecar
skipped: list of files that were skipped
skipped_album: list of tuples of (file, album) for any files that were skipped from an album
to_touch: list of files that were touched
touched: list of files that were touched
updated: list of files that were updated
xattr_skipped: list of files where xattr was skipped
xattr_written: list of files where xattr was written
user_written: list of files written by user post_function
user_skipped: list of files skipped by user post_function
user_error: list of tuples of (filename, error) for any errors generated by user post_function
Notes:
Each attribute is a list of files or None if no files for that attribute.
Error and warning attributes are a list of tuples of (filename, error) where filename is the file that caused the error and error is the error message.
Album attributes are a list of tuples of (file, album) where file is the file exported and album is the album it was exported to.
ExportResults can be added together with the += operator to combine results as the export progresses.
"""
local_vars = locals()
self._datetime = datetime.now().isoformat()
for attr in self.attributes:
@ -370,11 +419,14 @@ class ExportResults:
+ self.sidecar_user_written
+ self.sidecar_user_skipped
+ self.missing
+ self.user_written
+ self.user_skipped
)
files += [x[0] for x in self.exiftool_warning]
files += [x[0] for x in self.exiftool_error]
files += [x[0] for x in self.error]
files += [x[0] for x in self.sidecar_user_error]
files += [x[0] for x in self.user_error]
return list(set(files))

55
tests/test_cli.py
View File

@ -556,7 +556,7 @@ CLI_EXPORTED_FILENAME_TEMPLATE_LONG_DESCRIPTION = [
"pellentesque eu, pretium q.tif"
]
CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B"
CLI_EXPORT_UUID = "D79B8D77-BFFC-460B-9312-034F2877D35B" # Pumkins2.jpg
CLI_EXPORT_UUID_STATUE = "3DD2C897-F19E-4CA6-8C22-B027D5A71907"
CLI_EXPORT_UUID_KEYWORD_PATHSEP = "7783E8E6-9CAC-40F3-BE22-81FB7051C266"
CLI_EXPORT_UUID_LONG_DESCRIPTION = "8846E3E6-8AC8-4857-8448-E3D025784410"
@ -8823,7 +8823,7 @@ def test_export_post_function_exception():
"-V",
],
)
assert result.exit_code == 0
assert result.exit_code != 0
assert "Error running post-function" in result.output
@ -8862,6 +8862,57 @@ def test_export_post_function_bad_value():
assert "Could not load function" in result.output
def test_export_post_function_results():
"""Test --post-function with returned ExportResults, uses the post_function in examples/post_function.py"""
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
tempdir = os.getcwd()
result = runner.invoke(
cli_main,
[
"export",
".",
"--db",
os.path.join(cwd, PHOTOS_DB_15_7),
"--name",
"wedding",
"-V",
"--post-function",
f"{cwd}/examples/post_function.py::post_function",
],
)
assert result.exit_code == 0
assert "Writing new file" in result.output
export_files = glob.glob(os.path.join(f"{tempdir}", "*"))
assert os.path.join(tempdir, "wedding.jpg.new") in export_files
# run again with --update and --cleanup
result = runner.invoke(
cli_main,
[
"export",
".",
"--db",
os.path.join(cwd, PHOTOS_DB_15_7),
"--name",
"wedding",
"-V",
"--post-function",
f"{cwd}/examples/post_function.py::post_function",
"--update",
"--cleanup",
],
)
assert result.exit_code == 0
assert "Skipping file" in result.output
assert "Deleted: 0 files" in result.output
export_files = glob.glob(os.path.join(f"{tempdir}", "*"))
assert os.path.join(tempdir, "wedding.jpg.new") in export_files
def test_export_directory_template_function():
"""Test --directory with template function"""