forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial implementation for issue #265

Browse Source
This commit is contained in:
Rhet Turnbull
2020-11-26 09:08:26 -08:00
parent a807894095
commit 382fca3f92
4 changed files with 315 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
osxphotos/__main__.py
View File

@@ -1720,6 +1720,8 @@ def export(
results_skipped = []
results_exif_updated = []
results_touched = []
results_sidecar_json = []
results_sidecar_xmp = []
if verbose_:
for p in photos:
results = export_photo(
@@ -1762,6 +1764,8 @@ def export(
results_skipped.extend(results.skipped)
results_exif_updated.extend(results.exif_updated)
results_touched.extend(results.touched)
results_sidecar_json.extend(results.sidecar_json)
results_sidecar_xmp.extend(results.sidecar_xmp)
# if convert_to_jpeg and p.isphoto and p.uti != "public.jpeg":
# for photo_file in set(
@@ -1813,9 +1817,20 @@ def export(
results_skipped.extend(results.skipped)
results_exif_updated.extend(results.exif_updated)
results_touched.extend(results.touched)
results_sidecar_json.extend(results.sidecar_json)
results_sidecar_xmp.extend(results.sidecar_xmp)
stop_time = time.perf_counter()
# print summary results
# print(f"results_exported: {results_exported}")
# print(f"results_new: {results_new}")
# print(f"results_updated: {results_updated}")
# print(f"results_skipped: {results_skipped}")
# print(f"results_exif_updated: {results_exif_updated}")
# print(f"results_touched: {results_touched}")
# print(f"results_sidecar_json: {results_sidecar_json}")
# print(f"results_sidecar_xmp: {results_sidecar_xmp}")
if update:
photo_str_new = "photos" if len(results_new) != 1 else "photo"
photo_str_updated = "photos" if len(results_updated) != 1 else "photo"
@@ -2364,19 +2379,19 @@ def export_photo(
if photo.ismissing:
space = " " if not verbose_ else ""
verbose(f"{space}Skipping missing photo {photo.original_filename}")
return ExportResults([], [], [], [], [], [])
return ExportResults([], [], [], [], [], [], [], [])
elif photo.path is None:
space = " " if not verbose_ else ""
verbose(
f"{space}WARNING: photo {photo.original_filename} ({photo.uuid}) is missing but ismissing=False, "
f"skipping {photo.original_filename}"
)
return ExportResults([], [], [], [], [], [])
return ExportResults([], [], [], [], [], [], [], [])
elif photo.ismissing and not photo.iscloudasset and not photo.incloud:
verbose(
f"Skipping missing {photo.original_filename}: not iCloud asset or missing from cloud"
)
return ExportResults([], [], [], [], [], [])
return ExportResults([], [], [], [], [], [], [], [])
results_exported = []
results_new = []
@@ -2384,6 +2399,8 @@ def export_photo(
results_skipped = []
results_exif_updated = []
results_touched = []
results_sidecar_json = []
results_sidecar_xmp = []
export_original = not (skip_original_if_edited and photo.hasadjustments)
@@ -2459,6 +2476,7 @@ def export_photo(
jpeg_quality=jpeg_quality,
ignore_date_modified=ignore_date_modified,
use_photokit=use_photokit,
verbose=verbose,
)
results_exported.extend(export_results.exported)
@@ -2467,6 +2485,8 @@ def export_photo(
results_skipped.extend(export_results.skipped)
results_exif_updated.extend(export_results.exif_updated)
results_touched.extend(export_results.touched)
results_sidecar_json.extend(export_results.sidecar_json)
results_sidecar_xmp.extend(export_results.sidecar_xmp)
if verbose_:
for exported in export_results.exported:
@@ -2522,6 +2542,7 @@ def export_photo(
jpeg_quality=jpeg_quality,
ignore_date_modified=ignore_date_modified,
use_photokit=use_photokit,
verbose=verbose,
)
results_exported.extend(export_results_edited.exported)
@@ -2530,6 +2551,8 @@ def export_photo(
results_skipped.extend(export_results_edited.skipped)
results_exif_updated.extend(export_results_edited.exif_updated)
results_touched.extend(export_results_edited.touched)
results_sidecar_json.extend(export_results_edited.sidecar_json)
results_sidecar_xmp.extend(export_results_edited.sidecar_xmp)
if verbose_:
for exported in export_results_edited.exported:
@@ -2550,6 +2573,8 @@ def export_photo(
results_skipped,
results_exif_updated,
results_touched,
results_sidecar_json,
results_sidecar_xmp,
)
@@ -2579,7 +2604,11 @@ def get_filenames_from_template(photo, filename_template, original_name):
)
filenames = [f"{file_}{photo_ext}" for file_ in filenames]
else:
filenames = [photo.original_filename] if (original_name and (photo.original_filename is not None)) else [photo.filename]
filenames = (
[photo.original_filename]
if (original_name and (photo.original_filename is not None))
else [photo.filename]
)
filenames = [sanitize_filename(filename) for filename in filenames]
return filenames

133
osxphotos/export_db.py
View File

@@ -14,7 +14,7 @@ from sqlite3 import Error
from ._version import __version__
OSXPHOTOS_EXPORTDB_VERSION = "2.0"
OSXPHOTOS_EXPORTDB_VERSION = "3.0"
class ExportDB_ABC(ABC):
@@ -76,6 +76,22 @@ class ExportDB_ABC(ABC):
def set_exifdata_for_file(self, uuid, exifdata):
pass
@abstractmethod
def set_exiftool_json_sidecar_for_file(self, filename, stats):
pass
@abstractmethod
def get_exiftool_json_sidecar_for_file(self, filename):
pass
@abstractmethod
def set_xmp_sidecar_for_file(self, filename, stats):
pass
@abstractmethod
def get_xmp_sidecar_for_file(self, filename):
pass
@abstractmethod
def set_data(
self,
@@ -87,6 +103,8 @@ class ExportDB_ABC(ABC):
edited_stat,
info_json,
exif_json,
exiftool_json_sidecar,
xmp_sidecar,
):
pass
@@ -141,6 +159,18 @@ class ExportDBNoOp(ExportDB_ABC):
def set_exifdata_for_file(self, uuid, exifdata):
pass
def set_exiftool_json_sidecar_for_file(self, filename, stats):
pass
def get_exiftool_json_sidecar_for_file(self, filename):
pass
def set_xmp_sidecar_for_file(self, filename, stats):
pass
def get_xmp_sidecar_for_file(self, filename):
pass
def set_data(
self,
filename,
@@ -151,6 +181,8 @@ class ExportDBNoOp(ExportDB_ABC):
edited_stat,
info_json,
exif_json,
exiftool_json_sidecar,
xmp_sidecar,
):
pass
@@ -379,6 +411,70 @@ class ExportDB(ExportDB_ABC):
except Error as e:
logging.warning(e)
def get_exiftool_json_sidecar_for_file(self, filename):
""" returns the exiftool JSON sidecar data for a file """
filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
conn = self._conn
try:
c = conn.cursor()
c.execute(
"SELECT sidecar_data FROM exiftool_json_sidecar WHERE filepath_normalized = ?",
(filename,),
)
results = c.fetchone()
sidecar_data = results[0] if results else None
except Error as e:
logging.warning(e)
sidecar_data = None
return sidecar_data
def set_exiftool_json_sidecar_for_file(self, filename, sidecar_data):
""" sets the exiftool JSON sidecar data for a file """
filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
conn = self._conn
try:
c = conn.cursor()
c.execute(
"INSERT OR REPLACE INTO exiftool_json_sidecar(filepath_normalized, sidecar_data) VALUES (?, ?);",
(filename, sidecar_data),
)
conn.commit()
except Error as e:
logging.warning(e)
def get_xmp_sidecar_for_file(self, filename):
""" returns the XMP sidecar data for a file """
filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
conn = self._conn
try:
c = conn.cursor()
c.execute(
"SELECT sidecar_data FROM xmp_sidecar WHERE filepath_normalized = ?",
(filename,),
)
results = c.fetchone()
sidecar_data = results[0] if results else None
except Error as e:
logging.warning(e)
sidecar_data = None
return sidecar_data
def set_xmp_sidecar_for_file(self, filename, sidecar_data):
""" sets the XMP sidecar data for a file """
filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
conn = self._conn
try:
c = conn.cursor()
c.execute(
"INSERT OR REPLACE INTO xmp_sidecar(filepath_normalized, sidecar_data) VALUES (?, ?);",
(filename, sidecar_data),
)
conn.commit()
except Error as e:
logging.warning(e)
def set_data(
self,
filename,
@@ -389,6 +485,8 @@ class ExportDB(ExportDB_ABC):
edited_stat,
info_json,
exif_json,
exiftool_json_sidecar,
xmp_sidecar,
):
""" sets all the data for file and uuid at once
"""
@@ -429,6 +527,14 @@ class ExportDB(ExportDB_ABC):
"INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
(filename_normalized, exif_json),
)
c.execute(
"INSERT OR REPLACE INTO exiftool_json_sidecar(filepath_normalized, sidecar_data) VALUES (?, ?);",
(filename_normalized, exiftool_json_sidecar),
)
c.execute(
"INSERT OR REPLACE INTO xmp_sidecar(filepath_normalized, sidecar_data) VALUES (?, ?);",
(filename_normalized, xmp_sidecar),
)
conn.commit()
except Error as e:
logging.warning(e)
@@ -479,13 +585,11 @@ class ExportDB(ExportDB_ABC):
if not os.path.isfile(dbfile):
conn = self._get_db_connection(dbfile)
if conn:
self._create_db_tables(conn)
self.was_created = True
self.was_upgraded = ()
self.version = OSXPHOTOS_EXPORTDB_VERSION
else:
if not conn:
raise Exception("Error getting connection to database {dbfile}")
self._create_db_tables(conn)
self.was_created = True
self.was_upgraded = ()
else:
conn = self._get_db_connection(dbfile)
self.was_created = False
@@ -495,8 +599,7 @@ class ExportDB(ExportDB_ABC):
self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
else:
self.was_upgraded = ()
self.version = OSXPHOTOS_EXPORTDB_VERSION
self.version = OSXPHOTOS_EXPORTDB_VERSION
return conn
def _get_db_connection(self, dbfile):
@@ -570,11 +673,23 @@ class ExportDB(ExportDB_ABC):
size INTEGER,
mtime REAL
); """,
"sql_xmp_table": """ CREATE TABLE IF NOT EXISTS xmp_sidecar (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
sidecar_data TEXT
); """,
"sql_exiftool_json_table": """ CREATE TABLE IF NOT EXISTS exiftool_json_sidecar (
id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL,
sidecar_data JSON
); """,
"sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
"sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
"sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
"sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
"sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
"sql_xmp_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_xmp_filename on xmp_sidecar (filepath_normalized);""",
"sql_exiftool_json_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exiftool_json_filename on exiftool_json_sidecar (filepath_normalized);""",
}
try:
c = conn.cursor()

132
osxphotos/photoinfo/_photoinfo_export.py
View File

@@ -13,6 +13,7 @@
# TODO: should this be its own PhotoExporter class?
import glob
import hashlib
import json
import logging
import os
@@ -41,14 +42,22 @@ from ..photokit import (
PhotoLibrary,
PhotoKitFetchFailed,
)
from ..utils import dd_to_dms_str, findfiles
from ..utils import dd_to_dms_str, findfiles, noop
ExportResults = namedtuple(
"ExportResults",
["exported", "new", "updated", "skipped", "exif_updated", "touched"],
["exported", "new", "updated", "skipped", "exif_updated", "touched", "sidecar_json", "sidecar_xmp"],
)
# hexdigest is not a class method, don't import this into PhotoInfo
def hexdigest(strval):
""" hexdigest of a string, using blake2b """
h = hashlib.blake2b(digest_size=20)
h.update(bytes(strval, "utf-8"))
return h.hexdigest()
# _export_photo_uuid_applescript is not a class method, don't import this into PhotoInfo
def _export_photo_uuid_applescript(
uuid,
@@ -321,6 +330,7 @@ def export2(
jpeg_quality=1.0,
ignore_date_modified=False,
use_photokit=False,
verbose=None,
):
""" export photo, like export but with update and dry_run options
dest: must be valid destination path or exception raised
@@ -364,6 +374,7 @@ def export2(
convert_to_jpeg: boolean; if True, converts non-jpeg images to jpeg
jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0. A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
ignore_date_modified: for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
verbose: optional callable function to use for printing verbose text during processing; if None (default), does not print output.
Returns: ExportResults namedtuple with fields: exported, new, updated, skipped
where each field is a list of file paths
@@ -380,6 +391,12 @@ def export2(
if export_db is None:
export_db = ExportDBNoOp()
if verbose is None:
verbose = noop
elif not callable(verbose):
raise TypeError("verbose must be callable")
self._verbose = verbose
# suffix to add to edited files
# e.g. name will be filename_edited.jpg
edited_identifier = "_edited"
@@ -501,14 +518,16 @@ def export2(
# might be exporting into a pre-ExportDB folder or the DB got deleted
dest_uuid = self.uuid
export_db.set_data(
dest,
self.uuid,
fileutil.file_sig(dest),
(None, None, None),
(None, None, None),
(None, None, None),
self.json(),
None,
filename=dest,
uuid=self.uuid,
orig_stat=fileutil.file_sig(dest),
exif_stat=(None, None, None),
converted_stat=(None, None, None),
edited_stat=(None, None, None),
info_json=self.json(),
exif_json=None,
exiftool_json_sidecar=None,
xmp_sidecar=None,
)
if dest_uuid != self.uuid:
# not the right file, find the right one
@@ -527,14 +546,16 @@ def export2(
dest = pathlib.Path(file_)
found_match = True
export_db.set_data(
dest,
self.uuid,
fileutil.file_sig(dest),
(None, None, None),
(None, None, None),
(None, None, None),
self.json(),
None,
filename=dest,
uuid=self.uuid,
orig_stat=fileutil.file_sig(dest),
exif_stat=(None, None, None),
converted_stat=(None, None, None),
edited_stat=(None, None, None),
info_json=self.json(),
exif_json=None,
exiftool_json_sidecar=None,
xmp_sidecar=None,
)
break
@@ -722,8 +743,10 @@ def export2(
)
# export metadata
sidecar_json_files = []
if sidecar_json:
sidecar_filename = dest.parent / pathlib.Path(f"{dest.stem}{dest.suffix}.json")
sidecar_json_files.append(str(sidecar_filename))
sidecar_str = self._exiftool_json_sidecar(
use_albums_as_keywords=use_albums_as_keywords,
use_persons_as_keywords=use_persons_as_keywords,
@@ -731,15 +754,29 @@ def export2(
description_template=description_template,
ignore_date_modified=ignore_date_modified,
)
if not dry_run:
try:
sidecar_digest = hexdigest(sidecar_str)
old_sidecar_digest = export_db.get_exiftool_json_sidecar_for_file(
sidecar_filename
)
write_sidecar = (
not update
or (update and not sidecar_filename.exists())
or (update and sidecar_digest != old_sidecar_digest)
)
if write_sidecar:
verbose(f"Writing exiftool JSON sidecar {sidecar_filename}")
if not dry_run:
self._write_sidecar(sidecar_filename, sidecar_str)
except Exception as e:
logging.warning(f"Error writing json sidecar to {sidecar_filename}")
raise e
export_db.set_exiftool_json_sidecar_for_file(
sidecar_filename, sidecar_digest
)
else:
verbose(f"Skipped up to date exiftool JSON sidecar {sidecar_filename}")
sidecar_xmp_files = []
if sidecar_xmp:
sidecar_filename = dest.parent / pathlib.Path(f"{dest.stem}{dest.suffix}.xmp")
sidecar_xmp_files.append(str(sidecar_filename))
sidecar_str = self._xmp_sidecar(
use_albums_as_keywords=use_albums_as_keywords,
use_persons_as_keywords=use_persons_as_keywords,
@@ -747,12 +784,20 @@ def export2(
description_template=description_template,
extension=dest.suffix[1:] if dest.suffix else None,
)
if not dry_run:
try:
sidecar_digest = hexdigest(sidecar_str)
old_sidecar_digest = export_db.get_xmp_sidecar_for_file(sidecar_filename)
write_sidecar = (
not update
or (update and not sidecar_filename.exists())
or (update and sidecar_digest != old_sidecar_digest)
)
if write_sidecar:
verbose(f"Writing XMP sidecar {sidecar_filename}")
if not dry_run:
self._write_sidecar(sidecar_filename, sidecar_str)
except Exception as e:
logging.warning(f"Error writing xmp sidecar to {sidecar_filename}")
raise e
export_db.set_xmp_sidecar_for_file(sidecar_filename, sidecar_digest)
else:
verbose(f"Skipped up to date XMP sidecar {sidecar_filename}")
# if exiftool, write the metadata
if update:
@@ -782,6 +827,7 @@ def export2(
if old_data is None or files_are_different:
# didn't have old data, assume we need to write it
# or files were different
verbose(f"Writing metadata with exiftool for {exported_file}")
if not dry_run:
self._write_exif_data(
exported_file,
@@ -805,8 +851,11 @@ def export2(
exported_file, fileutil.file_sig(exported_file)
)
exif_files_updated.append(exported_file)
else:
verbose(f"Skipped up to date exiftool metadata for {exported_file}")
elif exiftool and exif_files:
for exported_file in exif_files:
verbose(f"Writing metadata with exiftool for {exported_file}")
if not dry_run:
self._write_exif_data(
exported_file,
@@ -834,6 +883,7 @@ def export2(
if touch_file:
for exif_file in exif_files_updated:
verbose(f"Updating file modification time for {exif_file}")
touched_files.append(exif_file)
ts = int(self.date.timestamp())
fileutil.utime(exif_file, (ts, ts))
@@ -847,6 +897,8 @@ def export2(
update_skipped_files,
exif_files_updated,
touched_files,
sidecar_json_files,
sidecar_xmp_files,
)
return results
@@ -996,14 +1048,16 @@ def _export_photo(
fileutil.copy(src, dest_str, norsrc=no_xattr)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
converted_stat,
edited_stat,
self.json(),
None,
filename=dest_str,
uuid=self.uuid,
orig_stat=fileutil.file_sig(dest_str),
exif_stat=(None, None, None),
converted_stat=converted_stat,
edited_stat=edited_stat,
info_json=self.json(),
exif_json=None,
exiftool_json_sidecar=None,
xmp_sidecar=None,
)
if touched_files:
@@ -1017,6 +1071,8 @@ def _export_photo(
update_skipped_files,
[],
touched_files,
[],
[],
)
@@ -1087,9 +1143,9 @@ def _exiftool_dict(
IPTC:Keywords (may include album name, person name, or template)
XMP:Subject
XMP:PersonInImage
EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
EXIF:GPSLatitude, EXIF:GPSLongitude
EXIF:GPSPosition
EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
EXIF:DateTimeOriginal
EXIF:OffsetTimeOriginal
EXIF:ModifyDate
@@ -1249,9 +1305,9 @@ def _exiftool_json_sidecar(
IPTC:Keywords (may include album name, person name, or template)
XMP:Subject
XMP:PersonInImage
EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
EXIF:GPSLatitude, EXIF:GPSLongitude
EXIF:GPSPosition
EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
EXIF:DateTimeOriginal
EXIF:OffsetTimeOriginal
EXIF:ModifyDate