Working on making export CLI threadsafe
This commit is contained in:
Rhet Turnbull
@@ -1,5 +1,7 @@
|
|||||||
"""export command for osxphotos CLI"""
|
"""export command for osxphotos CLI"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import atexit
|
import atexit
|
||||||
import inspect
|
import inspect
|
||||||
import os
|
import os
|
||||||
@@ -9,7 +11,8 @@ import shlex
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from typing import Iterable, List, Optional, Tuple
|
from typing import Iterable, List, Optional, Tuple, Any, Callable
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
import click
|
import click
|
||||||
from osxmetadata import (
|
from osxmetadata import (
|
||||||
@@ -1426,173 +1429,156 @@ def export(
|
|||||||
|
|
||||||
photo_num = 0
|
photo_num = 0
|
||||||
num_exported = 0
|
num_exported = 0
|
||||||
|
# hack to avoid passing all the options to export_photo
|
||||||
|
kwargs = locals().copy()
|
||||||
|
kwargs["export_dir"] = dest
|
||||||
|
kwargs["export_preview"] = preview
|
||||||
limit_str = f" (limit = [num]{limit}[/num])" if limit else ""
|
limit_str = f" (limit = [num]{limit}[/num])" if limit else ""
|
||||||
with rich_progress(console=get_verbose_console(), mock=no_progress) as progress:
|
with rich_progress(console=get_verbose_console(), mock=no_progress) as progress:
|
||||||
task = progress.add_task(
|
task = progress.add_task(
|
||||||
f"Exporting [num]{num_photos}[/] photos{limit_str}", total=num_photos
|
f"Exporting [num]{num_photos}[/] photos{limit_str}", total=num_photos
|
||||||
)
|
)
|
||||||
for p in photos:
|
futures = []
|
||||||
photo_num += 1
|
with concurrent.futures.ThreadPoolExecutor(
|
||||||
# hack to avoid passing all the options to export_photo
|
# max_workers=os.cpu_count()
|
||||||
kwargs = {
|
max_workers=1,
|
||||||
k: v
|
) as executor:
|
||||||
for k, v in locals().items()
|
for p in photos:
|
||||||
if k in inspect.getfullargspec(export_photo).args
|
photo_num += 1
|
||||||
}
|
kwargs["photo_num"] = photo_num
|
||||||
kwargs["photo"] = p
|
futures.append(executor.submit(export_worker, p, **kwargs))
|
||||||
kwargs["export_dir"] = dest
|
|
||||||
kwargs["export_preview"] = preview
|
for future in concurrent.futures.as_completed(futures):
|
||||||
export_results = export_photo(**kwargs)
|
p, export_results = future.result()
|
||||||
if post_function:
|
if album_export and export_results.exported:
|
||||||
for function in post_function:
|
try:
|
||||||
# post function is tuple of (function, filename.py::function_name)
|
album_export.add(p)
|
||||||
verbose(f"Calling post-function [bold]{function[1]}")
|
export_results.exported_album = [
|
||||||
if not dry_run:
|
(filename, album_export.name)
|
||||||
try:
|
for filename in export_results.exported
|
||||||
function[0](p, export_results, verbose)
|
]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
rich_echo_error(
|
click.secho(
|
||||||
f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
|
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_export.name}: {e}",
|
||||||
|
fg=CLI_COLOR_ERROR,
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if album_skipped and export_results.skipped:
|
||||||
|
try:
|
||||||
|
album_skipped.add(p)
|
||||||
|
export_results.skipped_album = [
|
||||||
|
(filename, album_skipped.name)
|
||||||
|
for filename in export_results.skipped
|
||||||
|
]
|
||||||
|
except Exception as e:
|
||||||
|
click.secho(
|
||||||
|
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_skipped.name}: {e}",
|
||||||
|
fg=CLI_COLOR_ERROR,
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if album_missing and export_results.missing:
|
||||||
|
try:
|
||||||
|
album_missing.add(p)
|
||||||
|
export_results.missing_album = [
|
||||||
|
(filename, album_missing.name)
|
||||||
|
for filename in export_results.missing
|
||||||
|
]
|
||||||
|
except Exception as e:
|
||||||
|
click.secho(
|
||||||
|
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_missing.name}: {e}",
|
||||||
|
fg=CLI_COLOR_ERROR,
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
results += export_results
|
||||||
|
|
||||||
|
# all photo files (not including sidecars) that are part of this export set
|
||||||
|
# used below for applying Finder tags, etc.
|
||||||
|
photo_files = set(
|
||||||
|
export_results.exported
|
||||||
|
+ export_results.new
|
||||||
|
+ export_results.updated
|
||||||
|
+ export_results.exif_updated
|
||||||
|
+ export_results.converted_to_jpeg
|
||||||
|
+ export_results.skipped
|
||||||
|
)
|
||||||
|
|
||||||
|
if finder_tag_keywords or finder_tag_template:
|
||||||
|
if dry_run:
|
||||||
|
for filepath in photo_files:
|
||||||
|
verbose(
|
||||||
|
f"Writing Finder tags to [filepath]{filepath}[/]"
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
run_post_command(
|
tags_written, tags_skipped = write_finder_tags(
|
||||||
photo=p,
|
p,
|
||||||
post_command=post_command,
|
photo_files,
|
||||||
export_results=export_results,
|
keywords=finder_tag_keywords,
|
||||||
export_dir=dest,
|
keyword_template=keyword_template,
|
||||||
dry_run=dry_run,
|
album_keyword=album_keyword,
|
||||||
exiftool_path=exiftool_path,
|
person_keyword=person_keyword,
|
||||||
export_db=export_db,
|
exiftool_merge_keywords=exiftool_merge_keywords,
|
||||||
verbose=verbose,
|
finder_tag_template=finder_tag_template,
|
||||||
)
|
strip=strip,
|
||||||
|
export_dir=dest,
|
||||||
if album_export and export_results.exported:
|
verbose=verbose,
|
||||||
try:
|
|
||||||
album_export.add(p)
|
|
||||||
export_results.exported_album = [
|
|
||||||
(filename, album_export.name)
|
|
||||||
for filename in export_results.exported
|
|
||||||
]
|
|
||||||
except Exception as e:
|
|
||||||
click.secho(
|
|
||||||
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_export.name}: {e}",
|
|
||||||
fg=CLI_COLOR_ERROR,
|
|
||||||
err=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
if album_skipped and export_results.skipped:
|
|
||||||
try:
|
|
||||||
album_skipped.add(p)
|
|
||||||
export_results.skipped_album = [
|
|
||||||
(filename, album_skipped.name)
|
|
||||||
for filename in export_results.skipped
|
|
||||||
]
|
|
||||||
except Exception as e:
|
|
||||||
click.secho(
|
|
||||||
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_skipped.name}: {e}",
|
|
||||||
fg=CLI_COLOR_ERROR,
|
|
||||||
err=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
if album_missing and export_results.missing:
|
|
||||||
try:
|
|
||||||
album_missing.add(p)
|
|
||||||
export_results.missing_album = [
|
|
||||||
(filename, album_missing.name)
|
|
||||||
for filename in export_results.missing
|
|
||||||
]
|
|
||||||
except Exception as e:
|
|
||||||
click.secho(
|
|
||||||
f"Error adding photo {p.original_filename} ({p.uuid}) to album {album_missing.name}: {e}",
|
|
||||||
fg=CLI_COLOR_ERROR,
|
|
||||||
err=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
results += export_results
|
|
||||||
|
|
||||||
# all photo files (not including sidecars) that are part of this export set
|
|
||||||
# used below for applying Finder tags, etc.
|
|
||||||
photo_files = set(
|
|
||||||
export_results.exported
|
|
||||||
+ export_results.new
|
|
||||||
+ export_results.updated
|
|
||||||
+ export_results.exif_updated
|
|
||||||
+ export_results.converted_to_jpeg
|
|
||||||
+ export_results.skipped
|
|
||||||
)
|
|
||||||
|
|
||||||
if finder_tag_keywords or finder_tag_template:
|
|
||||||
if dry_run:
|
|
||||||
for filepath in photo_files:
|
|
||||||
verbose(f"Writing Finder tags to [filepath]{filepath}[/]")
|
|
||||||
else:
|
|
||||||
tags_written, tags_skipped = write_finder_tags(
|
|
||||||
p,
|
|
||||||
photo_files,
|
|
||||||
keywords=finder_tag_keywords,
|
|
||||||
keyword_template=keyword_template,
|
|
||||||
album_keyword=album_keyword,
|
|
||||||
person_keyword=person_keyword,
|
|
||||||
exiftool_merge_keywords=exiftool_merge_keywords,
|
|
||||||
finder_tag_template=finder_tag_template,
|
|
||||||
strip=strip,
|
|
||||||
export_dir=dest,
|
|
||||||
verbose=verbose,
|
|
||||||
)
|
|
||||||
export_results.xattr_written.extend(tags_written)
|
|
||||||
export_results.xattr_skipped.extend(tags_skipped)
|
|
||||||
results.xattr_written.extend(tags_written)
|
|
||||||
results.xattr_skipped.extend(tags_skipped)
|
|
||||||
|
|
||||||
if xattr_template:
|
|
||||||
if dry_run:
|
|
||||||
for filepath in photo_files:
|
|
||||||
verbose(
|
|
||||||
f"Writing extended attributes to [filepath]{filepath}[/]"
|
|
||||||
)
|
)
|
||||||
else:
|
export_results.xattr_written.extend(tags_written)
|
||||||
xattr_written, xattr_skipped = write_extended_attributes(
|
export_results.xattr_skipped.extend(tags_skipped)
|
||||||
p,
|
results.xattr_written.extend(tags_written)
|
||||||
photo_files,
|
results.xattr_skipped.extend(tags_skipped)
|
||||||
xattr_template,
|
|
||||||
strip=strip,
|
|
||||||
export_dir=dest,
|
|
||||||
verbose=verbose,
|
|
||||||
)
|
|
||||||
export_results.xattr_written.extend(xattr_written)
|
|
||||||
export_results.xattr_skipped.extend(xattr_skipped)
|
|
||||||
results.xattr_written.extend(xattr_written)
|
|
||||||
results.xattr_skipped.extend(xattr_skipped)
|
|
||||||
|
|
||||||
report_writer.write(export_results)
|
if xattr_template:
|
||||||
|
if dry_run:
|
||||||
if print_template:
|
for filepath in photo_files:
|
||||||
options = RenderOptions(export_dir=dest)
|
verbose(
|
||||||
for template in print_template:
|
f"Writing extended attributes to [filepath]{filepath}[/]"
|
||||||
rendered_templates, unmatched = p.render_template(
|
)
|
||||||
template,
|
else:
|
||||||
options,
|
xattr_written, xattr_skipped = write_extended_attributes(
|
||||||
)
|
p,
|
||||||
if unmatched:
|
photo_files,
|
||||||
rich_click_echo(
|
xattr_template,
|
||||||
f"[warning]Unmatched template field: {unmatched}[/]"
|
strip=strip,
|
||||||
|
export_dir=dest,
|
||||||
|
verbose=verbose,
|
||||||
)
|
)
|
||||||
for rendered_template in rendered_templates:
|
export_results.xattr_written.extend(xattr_written)
|
||||||
if not rendered_template:
|
export_results.xattr_skipped.extend(xattr_skipped)
|
||||||
continue
|
results.xattr_written.extend(xattr_written)
|
||||||
rich_click_echo(rendered_template)
|
results.xattr_skipped.extend(xattr_skipped)
|
||||||
|
|
||||||
progress.advance(task)
|
report_writer.write(export_results)
|
||||||
|
|
||||||
# handle limit
|
if print_template:
|
||||||
if export_results.exported:
|
options = RenderOptions(export_dir=dest)
|
||||||
# if any photos were exported, increment num_exported used by limit
|
for template in print_template:
|
||||||
# limit considers each PhotoInfo object as a single photo even if multiple files are exported
|
rendered_templates, unmatched = p.render_template(
|
||||||
num_exported += 1
|
template,
|
||||||
if limit and num_exported >= limit:
|
options,
|
||||||
# advance progress to end
|
)
|
||||||
progress.advance(task, num_photos - photo_num)
|
if unmatched:
|
||||||
break
|
rich_click_echo(
|
||||||
|
f"[warning]Unmatched template field: {unmatched}[/]"
|
||||||
|
)
|
||||||
|
for rendered_template in rendered_templates:
|
||||||
|
if not rendered_template:
|
||||||
|
continue
|
||||||
|
rich_click_echo(rendered_template)
|
||||||
|
|
||||||
|
progress.advance(task)
|
||||||
|
|
||||||
|
# handle limit
|
||||||
|
if export_results.exported:
|
||||||
|
# if any photos were exported, increment num_exported used by limit
|
||||||
|
# limit considers each PhotoInfo object as a single photo even if multiple files are exported
|
||||||
|
num_exported += 1
|
||||||
|
if limit and num_exported >= limit:
|
||||||
|
# advance progress to end
|
||||||
|
progress.advance(task, num_photos - photo_num)
|
||||||
|
break
|
||||||
|
|
||||||
photo_str_total = pluralize(len(photos), "photo", "photos")
|
photo_str_total = pluralize(len(photos), "photo", "photos")
|
||||||
if update or force_update:
|
if update or force_update:
|
||||||
@@ -1682,6 +1668,45 @@ def export(
|
|||||||
export_db.close()
|
export_db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def export_worker(
|
||||||
|
photo: osxphotos.PhotoInfo, **kwargs
|
||||||
|
) -> tuple[osxphotos.PhotoInfo, ExportResults]:
|
||||||
|
"""Export worker function for multi-threaded export of photos"""
|
||||||
|
dry_run = kwargs["dry_run"]
|
||||||
|
verbose: Callable[[str], Any] = kwargs["verbose"]
|
||||||
|
export_args = {
|
||||||
|
k: v
|
||||||
|
for k, v in kwargs.items()
|
||||||
|
if k in inspect.getfullargspec(export_photo).args
|
||||||
|
}
|
||||||
|
export_args["photo"] = photo
|
||||||
|
export_results = export_photo(**export_args)
|
||||||
|
if post_function := kwargs["post_function"]:
|
||||||
|
for function in post_function:
|
||||||
|
# post function is tuple of (function, filename.py::function_name)
|
||||||
|
verbose(f"Calling post-function [bold]{function[1]}")
|
||||||
|
if not dry_run:
|
||||||
|
try:
|
||||||
|
function[0](photo, export_results, verbose)
|
||||||
|
except Exception as e:
|
||||||
|
rich_echo_error(
|
||||||
|
f"[error]Error running post-function [italic]{function[1]}[/italic]: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
|
run_post_command(
|
||||||
|
photo=photo,
|
||||||
|
post_command=kwargs["post_command"],
|
||||||
|
export_results=export_results,
|
||||||
|
export_dir=kwargs["dest"],
|
||||||
|
dry_run=dry_run,
|
||||||
|
exiftool_path=kwargs["exiftool_path"],
|
||||||
|
export_db=kwargs["export_db"],
|
||||||
|
verbose=verbose,
|
||||||
|
)
|
||||||
|
|
||||||
|
return photo, export_results
|
||||||
|
|
||||||
|
|
||||||
def export_photo(
|
def export_photo(
|
||||||
photo=None,
|
photo=None,
|
||||||
dest=None,
|
dest=None,
|
||||||
|
|||||||
Reference in New Issue
Block a user