From 02b6698c80903701cb56f6f8c50f1f3687be55dc Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Mon, 24 Jul 2023 06:38:44 -0700 Subject: [PATCH] Feature custom sidecar 1123 (#1129) * Working on custom sidecar, #1123 * Working on custom sidecar, #1123 * Added custom sidecar example * Added WRITE_SKIPPED argument for --sidecar-template * Updated report writer for user sidecars * Added noqa to long lines in report writer * Initial tests for --sidecar-template * Initial tests for --sidecar-template * Initial tests for --sidecar-template * Completed tests for --sidecar-template * Added json example for --sidecar-template * Added json example for --sidecar-template --- examples/custom_sidecar.mako | 227 +----------- examples/custom_sidecar_json.mako | 6 + examples/custom_sidecar_xmp.mako | 223 ++++++++++++ osxphotos/cli/export.py | 68 +++- osxphotos/cli/param_types.py | 17 + osxphotos/cli/report_writer.py | 21 +- osxphotos/cli/sidecar.py | 144 ++++++++ osxphotos/photoexporter.py | 6 + tests/custom_sidecar.mako | 22 ++ tests/test_cli_export_sidecar_template.py | 403 ++++++++++++++++++++++ 10 files changed, 914 insertions(+), 223 deletions(-) create mode 100644 examples/custom_sidecar_json.mako create mode 100644 examples/custom_sidecar_xmp.mako create mode 100644 osxphotos/cli/sidecar.py create mode 100644 tests/custom_sidecar.mako create mode 100644 tests/test_cli_export_sidecar_template.py diff --git a/examples/custom_sidecar.mako b/examples/custom_sidecar.mako index b72880a3..62c9277a 100644 --- a/examples/custom_sidecar.mako +++ b/examples/custom_sidecar.mako @@ -1,223 +1,22 @@ <%doc> - This is an example Mako template for use with --sidecar-template which produces an XMP sidecar file + This is an example Mako template for use with --sidecar-template For more information on Mako templates, see https://docs.makotemplates.org/en/latest/ The template will be passed three variables for rendering: - photo: a PhotoInfo object for the photo being exported - photo_path: a pathlib.Path object for the photo file being exported - sidecar_path: a pathlib.Path object for the sidecar file being written + photo: a PhotoInfo object for the photo being exported + photo_path: a pathlib.Path object for the photo file being exported + sidecar_path: a pathlib.Path object for the sidecar file being written -<%def name="photoshop_sidecar_for_extension(extension)"> - % if extension is None: - +<%def name="rating(photo)" filter="trim">\ + % if photo.favorite: + ★★★★★ % else: - ${extension} + ★☆☆☆☆ % endif - +\ -<%def name="dc_description(desc)"> - % if desc is None: - - - - - - % else: - - - ${desc | x} - - - % endif - - -<%def name="dc_title(title)"> - % if title is None: - - - - - - % else: - - - ${title | x} - - - % endif - - -<%def name="dc_subject(subject)"> - % if subject: - - - % for subj in subject: - ${subj | x} - % endfor - - - % endif - - -<%def name="dc_datecreated(date)"> - % if date is not None: - ${date.isoformat()} - % endif - - -<%def name="iptc_personinimage(persons)"> - % if persons: - - - % for person in persons: - ${person | x} - % endfor - - - % endif - - -<%def name="dk_tagslist(keywords)"> - % if keywords: - - - % for keyword in keywords: - ${keyword | x} - % endfor - - - % endif - - -<%def name="adobe_createdate(date)"> - % if date is not None: - ${date.strftime("%Y-%m-%dT%H:%M:%S")} - % endif - - -<%def name="adobe_modifydate(date)"> - % if date is not None: - ${date.strftime("%Y-%m-%dT%H:%M:%S")} - % endif - - -<%def name="xmp_rating(rating)"> - % if rating is not None: - ${rating} - % endif - - -<%def name="gps_info(latitude, longitude)"> - % if latitude is not None and longitude is not None: - ${int(abs(longitude))},${(abs(longitude) % 1) * 60}${"E" if longitude >= 0 else "W"} - ${int(abs(latitude))},${(abs(latitude) % 1) * 60}${"N" if latitude >= 0 else "S"} - % endif - - -<%def name="mwg_face_regions(photo)"> - % if photo.face_info: - - - ${photo.width if photo.orientation in [5, 6, 7, 8] else photo.height} - ${photo.height if photo.orientation in [5, 6, 7, 8] else photo.width} - pixel - - - - % for face in photo.face_info: - - - ${'{0:.6f}'.format(face.mwg_rs_area.h)} - ${'{0:.6f}'.format(face.mwg_rs_area.w)} - ${'{0:.6f}'.format(face.mwg_rs_area.x)} - ${'{0:.6f}'.format(face.mwg_rs_area.y)} - normalized - - ${face.name} - ${face.roll} - Face - - % endfor - - - - % endif - - -<%def name="mpri_face_regions(photo)"> - % if photo.face_info: - - - - % for face in photo.face_info: - - ${face.name} - ${'{0:.6f}'.format(face.mpri_reg_rect.x)}, ${'{0:.6f}'.format(face.mpri_reg_rect.y)}, ${'{0:.6f}'.format(face.mpri_reg_rect.h)}, ${'{0:.6f}'.format(face.mpri_reg_rect.w)} - - % endfor - - - - % endif - - - - - - - -<% - extension = photo_path.suffix[1:].lower() if photo_path.suffix else "" -%> - ${photoshop_sidecar_for_extension(extension)} - ${dc_description(photo.description)} - ${dc_title(photo.title)} -<% - subjects = photo.keywords + photo.persons -%> - ${dc_subject(subjects)} - ${dc_datecreated(photo.date)} - - - - ${iptc_personinimage(photo.persons)} - - - - ${dk_tagslist(photo.keywords)} - - - - ${adobe_createdate(photo.date)} - ${adobe_modifydate(photo.date)} - ${xmp_rating("5" if photo.favorite else None)} - - - - ${gps_info(*photo.location)} - - - - ${mwg_face_regions(photo)} - - - - ${mpri_face_regions(photo)} - - - - - \ No newline at end of file +Photo: ${photo_path.name} +UUID: ${photo.uuid} +Sidecar: ${sidecar_path.name} +Rating: ${rating(photo)} diff --git a/examples/custom_sidecar_json.mako b/examples/custom_sidecar_json.mako new file mode 100644 index 00000000..93baa937 --- /dev/null +++ b/examples/custom_sidecar_json.mako @@ -0,0 +1,6 @@ +<%doc> +Mako template to dump a full json representation of the photo object +Can be run from the command line with: +osxphotos export /path/to/export --sidecar-template custom_sidecar_json.mako "{filepath}.json" yes no yes + +${photo.json(shallow=False, indent=4)} \ No newline at end of file diff --git a/examples/custom_sidecar_xmp.mako b/examples/custom_sidecar_xmp.mako new file mode 100644 index 00000000..b72880a3 --- /dev/null +++ b/examples/custom_sidecar_xmp.mako @@ -0,0 +1,223 @@ +<%doc> + This is an example Mako template for use with --sidecar-template which produces an XMP sidecar file + For more information on Mako templates, see https://docs.makotemplates.org/en/latest/ + + The template will be passed three variables for rendering: + photo: a PhotoInfo object for the photo being exported + photo_path: a pathlib.Path object for the photo file being exported + sidecar_path: a pathlib.Path object for the sidecar file being written + + +<%def name="photoshop_sidecar_for_extension(extension)"> + % if extension is None: + + % else: + ${extension} + % endif + + +<%def name="dc_description(desc)"> + % if desc is None: + + + + + + % else: + + + ${desc | x} + + + % endif + + +<%def name="dc_title(title)"> + % if title is None: + + + + + + % else: + + + ${title | x} + + + % endif + + +<%def name="dc_subject(subject)"> + % if subject: + + + % for subj in subject: + ${subj | x} + % endfor + + + % endif + + +<%def name="dc_datecreated(date)"> + % if date is not None: + ${date.isoformat()} + % endif + + +<%def name="iptc_personinimage(persons)"> + % if persons: + + + % for person in persons: + ${person | x} + % endfor + + + % endif + + +<%def name="dk_tagslist(keywords)"> + % if keywords: + + + % for keyword in keywords: + ${keyword | x} + % endfor + + + % endif + + +<%def name="adobe_createdate(date)"> + % if date is not None: + ${date.strftime("%Y-%m-%dT%H:%M:%S")} + % endif + + +<%def name="adobe_modifydate(date)"> + % if date is not None: + ${date.strftime("%Y-%m-%dT%H:%M:%S")} + % endif + + +<%def name="xmp_rating(rating)"> + % if rating is not None: + ${rating} + % endif + + +<%def name="gps_info(latitude, longitude)"> + % if latitude is not None and longitude is not None: + ${int(abs(longitude))},${(abs(longitude) % 1) * 60}${"E" if longitude >= 0 else "W"} + ${int(abs(latitude))},${(abs(latitude) % 1) * 60}${"N" if latitude >= 0 else "S"} + % endif + + +<%def name="mwg_face_regions(photo)"> + % if photo.face_info: + + + ${photo.width if photo.orientation in [5, 6, 7, 8] else photo.height} + ${photo.height if photo.orientation in [5, 6, 7, 8] else photo.width} + pixel + + + + % for face in photo.face_info: + + + ${'{0:.6f}'.format(face.mwg_rs_area.h)} + ${'{0:.6f}'.format(face.mwg_rs_area.w)} + ${'{0:.6f}'.format(face.mwg_rs_area.x)} + ${'{0:.6f}'.format(face.mwg_rs_area.y)} + normalized + + ${face.name} + ${face.roll} + Face + + % endfor + + + + % endif + + +<%def name="mpri_face_regions(photo)"> + % if photo.face_info: + + + + % for face in photo.face_info: + + ${face.name} + ${'{0:.6f}'.format(face.mpri_reg_rect.x)}, ${'{0:.6f}'.format(face.mpri_reg_rect.y)}, ${'{0:.6f}'.format(face.mpri_reg_rect.h)}, ${'{0:.6f}'.format(face.mpri_reg_rect.w)} + + % endfor + + + + % endif + + + + + + + +<% + extension = photo_path.suffix[1:].lower() if photo_path.suffix else "" +%> + ${photoshop_sidecar_for_extension(extension)} + ${dc_description(photo.description)} + ${dc_title(photo.title)} +<% + subjects = photo.keywords + photo.persons +%> + ${dc_subject(subjects)} + ${dc_datecreated(photo.date)} + + + + ${iptc_personinimage(photo.persons)} + + + + ${dk_tagslist(photo.keywords)} + + + + ${adobe_createdate(photo.date)} + ${adobe_modifydate(photo.date)} + ${xmp_rating("5" if photo.favorite else None)} + + + + ${gps_info(*photo.location)} + + + + ${mwg_face_regions(photo)} + + + + ${mpri_face_regions(photo)} + + + + + \ No newline at end of file diff --git a/osxphotos/cli/export.py b/osxphotos/cli/export.py index 0090d1e3..65dced26 100644 --- a/osxphotos/cli/export.py +++ b/osxphotos/cli/export.py @@ -88,9 +88,10 @@ from .common import ( ) from .help import ExportCommand, get_help_msg from .list import _list_libraries -from .param_types import ExportDBType, FunctionCall, TemplateString +from .param_types import BooleanString, ExportDBType, FunctionCall, TemplateString from .report_writer import ReportWriterNoOp, export_report_writer_factory from .rich_progress import rich_progress +from .sidecar import generate_user_sidecar from .verbose import get_verbose_console, verbose_print @@ -304,7 +305,8 @@ from .verbose import get_verbose_console, verbose_print "The resulting file is named photoname.AAE. " "Note that to import these files back to Photos succesfully, you also need to " "export the edited photo and match the filename format Photos.app expects: " - "--filename 'IMG_{edited_version?E,}{id:04d}' --edited-suffix ''") + "--filename 'IMG_{edited_version?E,}{id:04d}' --edited-suffix ''", +) @click.option( "--sidecar", default=None, @@ -338,6 +340,41 @@ from .verbose import get_verbose_console, verbose_print "Warning: this may result in sidecar filename collisions if there are files of different " "types but the same name in the output directory, e.g. 'IMG_1234.JPG' and 'IMG_1234.MOV'.", ) +@click.option( + "--sidecar-template", + metavar="MAKO_TEMPLATE_FILE SIDECAR_FILENAME_TEMPLATE WRITE_SKIPPED STRIP_WHITESPACE STRIP_LINES", + multiple=True, + type=click.Tuple( + [ + click.Path(dir_okay=False, file_okay=True, exists=True), + TemplateString(), + BooleanString(), + BooleanString(), + BooleanString(), + ] + ), + help="Create a custom sidecar file for each photo exported with user provided Mako template (MAKO_TEMPLATE_FILE). " + "MAKO_TEMPLATE_FILE must be a valid Mako template (see https://www.makotemplates.org/). " + "The template will passed the following variables: photo (PhotoInfo object for the photo being exported), " + "sidecar_path (pathlib.Path object for the path to the sidecar being written), and " + "photo_path (pathlib.Path object for the path to the exported photo. " + "SIDECAR_TEMPLATE_FILENAME must be a valid template string (see Templating System in help) " + "which will be rendered to generate the filename of the sidecar file. " + "The `{filepath}` template variable may be used in the SIDECAR_TEMPLATE_FILENAME to refer to the filename of the " + "photo being exported. " + "WRITE_SKIPPED is a boolean value (true/false, yes/no, 1/0 are all valid values) and indicates whether or not " + "write the sidecar file even if the photo is skipped during export. " + "If WRITE_SKIPPED is false, the sidecar file will not be written if the photo is skipped during export. " + "If WRITE_SKIPPED is true, the sidecar file will be written even if the photo is skipped during export. " + "STRIP_WHITESPACE and STRIP_LINES are boolean values (true/false, yes/no, 1/0 are all valid values) " + "and indicate whether or not to strip whitespace and blank lines from the resulting sidecar file. " + "For example, to create a sidecar file with extension .xmp using a template file named 'sidecar.mako' " + "and write a sidecar for skipped photos and strip blank lines but not whitespace: " + "`--sidecar-template sidecar.mako '{filepath}.xmp' yes no yes`. " + "To do the same but to drop the photo extension from the sidecar filename: " + "`--sidecar-template sidecar.mako '{filepath.parent}/{filepath.stem}.xmp' yes no yes --sidecar-drop-ext`. " + "For an example Mako file see https://raw.githubusercontent.com/RhetTbull/osxphotos/main/examples/custom_sidecar.mako", +) @click.option( "--exiftool", is_flag=True, @@ -863,6 +900,7 @@ def export( export_aae, sidecar, sidecar_drop_ext, + sidecar_template, skip_bursts, skip_edited, skip_live, @@ -1090,6 +1128,7 @@ def export( export_aae = cfg.export_aae sidecar = cfg.sidecar sidecar_drop_ext = cfg.sidecar_drop_ext + sidecar_template = cfg.sidecar_template skip_bursts = cfg.skip_bursts skip_edited = cfg.skip_edited skip_live = cfg.skip_live @@ -1473,6 +1512,20 @@ def export( kwargs["photo"] = p kwargs["photo_num"] = photo_num export_results = export_photo(**kwargs) + + # generate custom sidecars if needed + if sidecar_template: + export_results += generate_user_sidecar( + photo=p, + export_results=export_results, + sidecar_template=sidecar_template, + exiftool_path=exiftool_path, + export_dir=dest, + dry_run=dry_run, + verbose=verbose, + ) + + # run post functions if post_function: for function in post_function: # post function is tuple of (function, filename.py::function_name) @@ -1485,6 +1538,7 @@ def export( f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}" ) + # run post command run_post_command( photo=p, post_command=post_command, @@ -1667,6 +1721,8 @@ def export( + results.sidecar_exiftool_skipped + results.sidecar_xmp_written + results.sidecar_xmp_skipped + + results.sidecar_user_written + + results.sidecar_user_skipped # include missing so a file that was already in export directory # but was missing on --update doesn't get deleted # (better to have old version than none) @@ -1766,7 +1822,7 @@ def export_photo( num_photos=1, tmpdir=None, update_errors=False, -): +) -> ExportResults: """Helper function for export that does the actual export Args: @@ -1817,6 +1873,7 @@ def export_photo( use_photos_export: bool; if True forces the use of AppleScript to export even if photo not missing verbose: callable for verbose output tmpdir: optional str; temporary directory to use for export + Returns: list of path(s) of exported photo or None if photo was missing @@ -2189,7 +2246,7 @@ def export_photo_to_directory( use_photokit, verbose, tmpdir, -): +) -> ExportResults: """Export photo to directory dest_path""" results = ExportResults() @@ -2739,9 +2796,6 @@ def run_post_command( verbose, ): # todo: pass in RenderOptions from export? (e.g. so it contains strip, etc?) - # todo: need a shell_quote template type: - # {shell_quote,{filepath}/foo/bar} - # that quotes everything in the default value for category, command_template in post_command: files = getattr(export_results, category) for f in files: diff --git a/osxphotos/cli/param_types.py b/osxphotos/cli/param_types.py index c5ed0486..54c5477a 100644 --- a/osxphotos/cli/param_types.py +++ b/osxphotos/cli/param_types.py @@ -18,6 +18,7 @@ from osxphotos.utils import expand_and_validate_filepath, load_function __all__ = [ "BitMathSize", + "BooleanString", "DateOffset", "DateTimeISO8601", "DeprecatedPath", @@ -302,3 +303,19 @@ class Longitude(click.ParamType): self.fail( f"Invalid longitude {value}. Must be a floating point number between -180 and 180." ) + + +class BooleanString(click.ParamType): + """A boolean string in the format True/False, Yes/No, T/F, Y/N, 1/0 (case insensitive)""" + + name = "BooleanString" + + def convert(self, value, param, ctx): + if value.lower() in ["true", "yes", "t", "y", "1"]: + return True + elif value.lower() in ["false", "no", "f", "n", "0"]: + return False + else: + self.fail( + f"Invalid boolean string {value}. Must be one of True/False, Yes/No, T/F, Y/N, 1/0 (case insensitive)." + ) diff --git a/osxphotos/cli/report_writer.py b/osxphotos/cli/report_writer.py index 9cce384f..edc9d5fb 100644 --- a/osxphotos/cli/report_writer.py +++ b/osxphotos/cli/report_writer.py @@ -98,6 +98,7 @@ class ExportReportWriterCSV(ReportWriterABC): "cleanup_deleted_file", "cleanup_deleted_directory", "exported_album", + "sidecar_user", ] mode = "a" if append else "w" @@ -197,9 +198,9 @@ class ExportReportWriterSQLite(ReportWriterABC): cursor = self._conn.cursor() cursor.execute( "INSERT INTO report " - "(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id) " + "(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, report_id, sidecar_user) " # noqa "VALUES " - "(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id);", + "(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :report_id, :sidecar_user);", # noqa data, ) self._conn.commit() @@ -262,6 +263,13 @@ class ExportReportWriterSQLite(ReportWriterABC): self._conn.cursor().execute("ALTER TABLE report ADD COLUMN report_id TEXT;") self._conn.commit() + # migrate report table and add sidecar_user column if needed (#1123) + if "sidecar_user" not in sqlite_columns(self._conn, "report"): + self._conn.cursor().execute( + "ALTER TABLE report ADD COLUMN sidecar_user INTEGER;" + ) + self._conn.commit() + # create report_summary view c.execute( """ @@ -347,6 +355,7 @@ def prepare_export_results_for_writing( "cleanup_deleted_file": false, "cleanup_deleted_directory": false, "exported_album": "", + "sidecar_user": false, } for result in export_results.exported: @@ -421,6 +430,14 @@ def prepare_export_results_for_writing( for result, album in export_results.exported_album: all_results[str(result)]["exported_album"] = album + for result in export_results.sidecar_user_written: + all_results[str(result)]["sidecar_user"] = true + all_results[str(result)]["exported"] = true + + for result in export_results.sidecar_user_skipped: + all_results[str(result)]["sidecar_user"] = true + all_results[str(result)]["skipped"] = true + return all_results diff --git a/osxphotos/cli/sidecar.py b/osxphotos/cli/sidecar.py new file mode 100644 index 00000000..02302982 --- /dev/null +++ b/osxphotos/cli/sidecar.py @@ -0,0 +1,144 @@ +"""Generate custom sidecar files for use wit `osxphotos export` command and --sidecar-template option""" + +from __future__ import annotations + +import pathlib +from typing import Callable + +import click +from mako.template import Template + +from osxphotos.photoexporter import ExportResults +from osxphotos.photoinfo import PhotoInfo +from osxphotos.phototemplate import PhotoTemplate, RenderOptions + + +def generate_user_sidecar( + photo: PhotoInfo, + export_results: ExportResults, + sidecar_template: tuple[tuple[str, str, bool, bool]], + exiftool_path: str, + export_dir: str, + dry_run: bool, + verbose: Callable[..., None], +) -> ExportResults: + """Generate custom sidecar files for use with `osxphotos export` command and --sidecar-template option + + Args: + photo: PhotoInfo object for photo + export_results: ExportResults object + sidecar_template: tuple of (template_file, filename_template) for sidecar template + strip_sidecar: bool, strip whitespace and blank lines from sidecar + exiftool_path: str, path to exiftool + export_dir: str, path to export directory + dry_run: bool, if True, do not actually write sidecar files + verbose: Callable[..., None], verbose logging function + + Returns: + ExportResults object with sidecar_user_written and sidecar_user_skipped set + """ + sidecar_results = ExportResults() + for ( + template_file, + filename_template, + write_skipped, + strip_whitespace, + strip_lines, + ) in sidecar_template: + if not write_skipped: + # skip writing sidecar if photo not exported + # but if run with --update and --cleanup, a sidecar file may have been written + # in the past, so check if it exists and if so keep it + for filepath in export_results.skipped: + template_filename = _render_sidecar_filename( + photo=photo, + filepath=filepath, + filename_template=filename_template, + export_dir=export_dir, + exiftool_path=exiftool_path, + ) + if template_filename and pathlib.Path(template_filename).exists(): + verbose( + f"Skipping existing sidecar file [filepath]{template_filename}[/]" + ) + sidecar_results.sidecar_user_skipped.append(template_filename) + + # write sidecar files for exported and missing files (and skipped if write_skipped) + files_to_process = export_results.exported + export_results.missing + if write_skipped: + files_to_process += export_results.skipped + for filepath in files_to_process: + template_filename = _render_sidecar_filename( + photo=photo, + filepath=filepath, + filename_template=filename_template, + export_dir=export_dir, + exiftool_path=exiftool_path, + ) + if not template_filename: + raise click.BadOptionUsage( + f"Invalid SIDECAR_FILENAME_TEMPLATE for --sidecar-template '{filename_template}'" + ) + + verbose(f"Writing sidecar file [filepath]{template_filename}[/]") + _render_sidecar_and_write_data( + template_file=template_file, + photo=photo, + template_filename=template_filename, + filepath=filepath, + strip_whitespace=strip_whitespace, + strip_lines=strip_lines, + dry_run=dry_run, + ) + sidecar_results.sidecar_user_written.append(template_filename) + + return sidecar_results + + +def _render_sidecar_filename( + photo: PhotoInfo, + filepath: str, + filename_template: str, + export_dir: str, + exiftool_path: str, +): + """Render sidecar filename template""" + render_options = RenderOptions(export_dir=export_dir, filepath=filepath) + photo_template = PhotoTemplate(photo, exiftool_path=exiftool_path) + template_filename, _ = photo_template.render( + filename_template, options=render_options + ) + template_filename = template_filename[0] if template_filename else None + + return template_filename + + +def _render_sidecar_and_write_data( + template_file: str, + photo: PhotoInfo, + template_filename: str, + filepath: str, + strip_whitespace: bool, + strip_lines: bool, + dry_run: bool, +): + """Render sidecar template and write data to file""" + sidecar = Template(filename=template_file) + sidecar_data = sidecar.render( + photo=photo, + sidecar_path=pathlib.Path(template_filename), + photo_path=pathlib.Path(filepath), + ) + + if strip_whitespace: + # strip whitespace + sidecar_data = "\n".join(line.strip() for line in sidecar_data.split("\n")) + if strip_lines: + # strip blank lines + sidecar_data = "\n".join( + line for line in sidecar_data.split("\n") if line.strip() + ) + if not dry_run: + # write sidecar file + with open(template_filename, "w") as f: + f.write(sidecar_data) diff --git a/osxphotos/photoexporter.py b/osxphotos/photoexporter.py index 907e59e5..c821e806 100644 --- a/osxphotos/photoexporter.py +++ b/osxphotos/photoexporter.py @@ -290,6 +290,8 @@ class ExportResults: "sidecar_json_written", "sidecar_xmp_skipped", "sidecar_xmp_written", + "sidecar_user_written", + "sidecar_user_skipped", "skipped", "skipped_album", "to_touch", @@ -321,6 +323,8 @@ class ExportResults: sidecar_json_written=None, sidecar_xmp_skipped=None, sidecar_xmp_written=None, + sidecar_user_written=None, + sidecar_user_skipped=None, skipped=None, skipped_album=None, to_touch=None, @@ -361,6 +365,8 @@ class ExportResults: + self.sidecar_exiftool_skipped + self.sidecar_xmp_written + self.sidecar_xmp_skipped + + self.sidecar_user_written + + self.sidecar_user_skipped + self.missing ) files += [x[0] for x in self.exiftool_warning] diff --git a/tests/custom_sidecar.mako b/tests/custom_sidecar.mako new file mode 100644 index 00000000..1decc37e --- /dev/null +++ b/tests/custom_sidecar.mako @@ -0,0 +1,22 @@ +<%doc> + This is an example Mako template for use with --sidecar-template + For more information on Mako templates, see https://docs.makotemplates.org/en/latest/ + + The template will be passed three variables for rendering: + photo: a PhotoInfo object for the photo being exported + photo_path: a pathlib.Path object for the photo file being exported + sidecar_path: a pathlib.Path object for the sidecar file being written + + +<%def name="rating(photo)" filter="trim">\ + % if photo.favorite: + ★★★★★ + % else: + ★☆☆☆☆ + % endif +\ + +Sidecar: ${sidecar_path.name} + Photo: ${photo_path.name} + UUID: ${photo.uuid} + Rating: ${rating(photo)} diff --git a/tests/test_cli_export_sidecar_template.py b/tests/test_cli_export_sidecar_template.py new file mode 100644 index 00000000..f49a5b70 --- /dev/null +++ b/tests/test_cli_export_sidecar_template.py @@ -0,0 +1,403 @@ +"""Test export with --sidecar-template (#1123)""" + +import csv +import json +import os +import pathlib +import sqlite3 + +import pytest +from click.testing import CliRunner + +from osxphotos.cli import export + +PHOTOS_DB = "./tests/Test-10.15.7.photoslibrary" + +PHOTO_UUID = "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51" # wedding.jpg +SIDECAR_FILENAME = "wedding.jpg.txt" +SIDECAR_DATA = """ + + +Sidecar: wedding.jpg.txt + Photo: wedding.jpg + UUID: E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51 + Rating: ★★★★★ +""" + + +def test_export_sidecar_template_1(): + """test basic export with --sidecar-template""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + assert sidecar_data == SIDECAR_DATA + + +def test_export_sidecar_template_strip_whitespace(): + """test basic export with --sidecar-template and STRIP_WHITESPACE = True""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "yes", + "no", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + sidecar_expected = ( + "\n".join(line.strip() for line in SIDECAR_DATA.splitlines()) + "\n" + ) + assert sidecar_data == sidecar_expected + + +def test_export_sidecar_template_strip_lines(): + """test basic export with --sidecar-template and STRIP_LINES = True""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "yes", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + sidecar_expected = "\n".join( + line for line in SIDECAR_DATA.splitlines() if line.strip() + ) + assert sidecar_data == sidecar_expected + + +def test_export_sidecar_template_strip_lines_strip_whitespace(): + """test basic export with --sidecar-template and STRIP_LINES = True and STRIP_WHITESPACE = True""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "yes", + "yes", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + sidecar_expected = "\n".join( + line.strip() for line in SIDECAR_DATA.splitlines() if line.strip() + ) + assert sidecar_data == sidecar_expected + + +def test_export_sidecar_template_update_no(): + """test basic export with --sidecar-template and WRITE_SKIPPED = False, also test --cleanup""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + ], + ) + + # run export again, should not update sidecar + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + "--update", + "--cleanup", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + sidecar_expected = "\n".join( + line.strip() for line in SIDECAR_DATA.splitlines() if line.strip() + ) + assert sidecar_data == SIDECAR_DATA + assert "Skipping existing sidecar file" in result.output + assert "Deleted: 0 files, 0 directories" in result.output + + +def test_export_sidecar_template_update_ues(): + """test basic export with --sidecar-template and WRITE_SKIPPED = True, also test --cleanup""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + ], + ) + + # run export again, should not update sidecar + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "yes", + "no", + "no", + "--update", + "--cleanup", + ], + ) + assert result.exit_code == 0 + sidecar_file = pathlib.Path(SIDECAR_FILENAME) + assert sidecar_file.exists() + sidecar_data = sidecar_file.read_text() + sidecar_expected = "\n".join( + line.strip() for line in SIDECAR_DATA.splitlines() if line.strip() + ) + assert sidecar_data == SIDECAR_DATA + assert "Skipping existing sidecar file" not in result.output + assert "Writing sidecar file" in result.output + assert "Deleted: 0 files, 0 directories" in result.output + + +def test_export_sidecar_template_report_csv(): + """test basic export with --sidecar-template --report to csv""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + "--report", + "report.csv", + ], + ) + assert result.exit_code == 0 + + # verify report output + report_file = pathlib.Path("report.csv") + assert report_file.exists() + csvreader = csv.DictReader(report_file.open()) + assert "sidecar_user" in csvreader.fieldnames + + found_sidecar = 0 + for row in csvreader: # sourcery skip: no-loop-in-tests + # sidecar ends with .txt so verify report has sidecar_user = 1 + if row["filename"].endswith( + ".txt" + ): # sourcery skip: no-conditionals-in-tests + assert str(row["sidecar_user"]) == "1" + found_sidecar += 1 + else: + assert str(row["sidecar_user"]) == "0" + assert found_sidecar + + +def test_export_sidecar_template_report_json(): + """test basic export with --sidecar-template --report to json""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + "--report", + "report.json", + ], + ) + assert result.exit_code == 0 + + # read the json report output and verify it is correct + report_file = pathlib.Path("report.json") + assert report_file.exists() + report_data = json.loads(report_file.read_text()) + assert "sidecar_user" in report_data[0] + found_sidecar = 0 + for row in report_data: # sourcery skip: no-loop-in-tests + # sidecar ends with .txt so verify report has sidecar_user = 1 + if row["filename"].endswith( + ".txt" + ): # sourcery skip: no-conditionals-in-tests + assert row["sidecar_user"] + found_sidecar += 1 + else: + assert not row["sidecar_user"] + assert found_sidecar + + +def test_export_sidecar_template_report_db(): + """test basic export with --sidecar-template --report to sqlite db""" + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke( + export, + [ + "--library", + os.path.join(cwd, PHOTOS_DB), + ".", + "-V", + "--uuid", + PHOTO_UUID, + "--sidecar-template", + os.path.join(cwd, "tests", "custom_sidecar.mako"), + "{filepath}.txt", + "no", + "no", + "no", + "--report", + "report.db", + ], + ) + assert result.exit_code == 0 + + # read the report sqlite db and verify it is correct + report_file = pathlib.Path("report.db") + assert report_file.exists() + conn = sqlite3.connect(report_file) + c = conn.cursor() + c.execute("SELECT filename, sidecar_user FROM report") + rows = c.fetchall() + found_sidecar = 0 + for row in rows: # sourcery skip: no-loop-in-tests + # sidecar ends with .txt so verify report has sidecar_user = 1 + if row[0].endswith(".txt"): # sourcery skip: no-conditionals-in-tests + assert row[1] == 1 + found_sidecar += 1 + else: + assert row[1] == 0 + assert found_sidecar