Added tests for issue #265

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.36.23](https://github.com/RhetTbull/osxphotos/compare/v0.36.22...v0.36.23)
+
+> 26 November 2020
+
+- Fix for missing original_filename, issue #267 [`fa33218`](https://github.com/RhetTbull/osxphotos/commit/fa332186ab3cdbe1bfd6496ff29b652ef984a5f8)
+- version bump [`b5195f9`](https://github.com/RhetTbull/osxphotos/commit/b5195f9d2b81cf6737b65e3cd3793ea9b0da13eb)
+- Updated test [`aa2ebf5`](https://github.com/RhetTbull/osxphotos/commit/aa2ebf55bb50eec14f86a532334b376e407f4bbc)
+
 #### [v0.36.22](https://github.com/RhetTbull/osxphotos/compare/v0.36.21...v0.36.22)
 
 > 26 November 2020

osxphotos/__main__.py

@@ -1720,6 +1720,8 @@ def export(
         results_skipped = []
         results_exif_updated = []
         results_touched = []
+        results_sidecar_json = []
+        results_sidecar_xmp = []
         if verbose_:
             for p in photos:
                 results = export_photo(
@@ -1762,6 +1764,8 @@ def export(
                 results_skipped.extend(results.skipped)
                 results_exif_updated.extend(results.exif_updated)
                 results_touched.extend(results.touched)
+                results_sidecar_json.extend(results.sidecar_json)
+                results_sidecar_xmp.extend(results.sidecar_xmp)
 
                 # if convert_to_jpeg and p.isphoto and p.uti != "public.jpeg":
                 #     for photo_file in set(
@@ -1813,9 +1817,20 @@ def export(
                     results_skipped.extend(results.skipped)
                     results_exif_updated.extend(results.exif_updated)
                     results_touched.extend(results.touched)
+                    results_sidecar_json.extend(results.sidecar_json)
+                    results_sidecar_xmp.extend(results.sidecar_xmp)
 
         stop_time = time.perf_counter()
         # print summary results
+        # print(f"results_exported: {results_exported}")
+        # print(f"results_new: {results_new}")
+        # print(f"results_updated: {results_updated}")
+        # print(f"results_skipped: {results_skipped}")
+        # print(f"results_exif_updated: {results_exif_updated}")
+        # print(f"results_touched: {results_touched}")
+        # print(f"results_sidecar_json: {results_sidecar_json}")
+        # print(f"results_sidecar_xmp: {results_sidecar_xmp}")
+
         if update:
             photo_str_new = "photos" if len(results_new) != 1 else "photo"
             photo_str_updated = "photos" if len(results_updated) != 1 else "photo"
@@ -2364,19 +2379,19 @@ def export_photo(
         if photo.ismissing:
             space = " " if not verbose_ else ""
             verbose(f"{space}Skipping missing photo {photo.original_filename}")
-            return ExportResults([], [], [], [], [], [])
+            return ExportResults([], [], [], [], [], [], [], [])
         elif photo.path is None:
             space = " " if not verbose_ else ""
             verbose(
                 f"{space}WARNING: photo {photo.original_filename} ({photo.uuid}) is missing but ismissing=False, "
                 f"skipping {photo.original_filename}"
             )
-            return ExportResults([], [], [], [], [], [])
+            return ExportResults([], [], [], [], [], [], [], [])
     elif photo.ismissing and not photo.iscloudasset and not photo.incloud:
         verbose(
             f"Skipping missing {photo.original_filename}: not iCloud asset or missing from cloud"
         )
-        return ExportResults([], [], [], [], [], [])
+        return ExportResults([], [], [], [], [], [], [], [])
 
     results_exported = []
     results_new = []
@@ -2384,6 +2399,8 @@ def export_photo(
     results_skipped = []
     results_exif_updated = []
     results_touched = []
+    results_sidecar_json = []
+    results_sidecar_xmp = []
 
     export_original = not (skip_original_if_edited and photo.hasadjustments)
 
@@ -2459,6 +2476,7 @@ def export_photo(
                     jpeg_quality=jpeg_quality,
                     ignore_date_modified=ignore_date_modified,
                     use_photokit=use_photokit,
+                    verbose=verbose,
                 )
 
                 results_exported.extend(export_results.exported)
@@ -2467,6 +2485,8 @@ def export_photo(
                 results_skipped.extend(export_results.skipped)
                 results_exif_updated.extend(export_results.exif_updated)
                 results_touched.extend(export_results.touched)
+                results_sidecar_json.extend(export_results.sidecar_json)
+                results_sidecar_xmp.extend(export_results.sidecar_xmp)
 
                 if verbose_:
                     for exported in export_results.exported:
@@ -2522,6 +2542,7 @@ def export_photo(
                         jpeg_quality=jpeg_quality,
                         ignore_date_modified=ignore_date_modified,
                         use_photokit=use_photokit,
+                        verbose=verbose,
                     )
 
                     results_exported.extend(export_results_edited.exported)
@@ -2530,6 +2551,8 @@ def export_photo(
                     results_skipped.extend(export_results_edited.skipped)
                     results_exif_updated.extend(export_results_edited.exif_updated)
                     results_touched.extend(export_results_edited.touched)
+                    results_sidecar_json.extend(export_results_edited.sidecar_json)
+                    results_sidecar_xmp.extend(export_results_edited.sidecar_xmp)
 
                     if verbose_:
                         for exported in export_results_edited.exported:
@@ -2550,6 +2573,8 @@ def export_photo(
         results_skipped,
         results_exif_updated,
         results_touched,
+        results_sidecar_json,
+        results_sidecar_xmp,
     )
 
 
@@ -2579,7 +2604,11 @@ def get_filenames_from_template(photo, filename_template, original_name):
             )
         filenames = [f"{file_}{photo_ext}" for file_ in filenames]
     else:
-                filenames = [photo.original_filename] if (original_name and (photo.original_filename is not None)) else [photo.filename]
+        filenames = (
+            [photo.original_filename]
+            if (original_name and (photo.original_filename is not None))
+            else [photo.filename]
+        )
 
     filenames = [sanitize_filename(filename) for filename in filenames]
     return filenames

osxphotos/_version.py

@@ -1,4 +1,4 @@
 """ version info """
 
-__version__ = "0.36.23"
+__version__ = "0.36.24"
 

osxphotos/export_db.py

@@ -14,7 +14,7 @@ from sqlite3 import Error
 
 from ._version import __version__
 
-OSXPHOTOS_EXPORTDB_VERSION = "2.0"
+OSXPHOTOS_EXPORTDB_VERSION = "3.2"
 
 
 class ExportDB_ABC(ABC):
@@ -76,6 +76,14 @@ class ExportDB_ABC(ABC):
     def set_exifdata_for_file(self, uuid, exifdata):
         pass
 
+    @abstractmethod
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        pass
+
+    @abstractmethod
+    def get_sidecar_for_file(self, filename):
+        pass
+
     @abstractmethod
     def set_data(
         self,
@@ -141,6 +149,12 @@ class ExportDBNoOp(ExportDB_ABC):
     def set_exifdata_for_file(self, uuid, exifdata):
         pass
 
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        pass
+
+    def get_sidecar_for_file(self, filename):
+        return None, (None, None, None)
+
     def set_data(
         self,
         filename,
@@ -379,6 +393,48 @@ class ExportDB(ExportDB_ABC):
         except Error as e:
             logging.warning(e)
 
+    def get_sidecar_for_file(self, filename):
+        """ returns the sidecar data and signature for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT sidecar_data, mode, size, mtime FROM sidecar WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            if results:
+                sidecar_data = results[0]
+                sidecar_sig = (
+                    results[1],
+                    results[2],
+                    int(results[3]) if results[3] is not None else None,
+                )
+            else:
+                sidecar_data = None
+                sidecar_sig = (None, None, None)
+        except Error as e:
+            logging.warning(e)
+            sidecar_data = None
+            sidecar_sig = (None, None, None)
+
+        return sidecar_data, sidecar_sig
+
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        """ sets the sidecar data and signature for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "INSERT OR REPLACE INTO sidecar(filepath_normalized, sidecar_data, mode, size, mtime) VALUES (?, ?, ?, ?, ?);",
+                (filename, sidecar_data, *sidecar_sig),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
     def set_data(
         self,
         filename,
@@ -479,13 +535,11 @@ class ExportDB(ExportDB_ABC):
 
         if not os.path.isfile(dbfile):
             conn = self._get_db_connection(dbfile)
-            if conn:
-                self._create_db_tables(conn)
-                self.was_created = True
-                self.was_upgraded = ()
-                self.version = OSXPHOTOS_EXPORTDB_VERSION
-            else:
+            if not conn:
                 raise Exception("Error getting connection to database {dbfile}")
+            self._create_db_tables(conn)
+            self.was_created = True
+            self.was_upgraded = ()
         else:
             conn = self._get_db_connection(dbfile)
             self.was_created = False
@@ -495,8 +549,7 @@ class ExportDB(ExportDB_ABC):
                 self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
             else:
                 self.was_upgraded = ()
-            self.version = OSXPHOTOS_EXPORTDB_VERSION
-
+        self.version = OSXPHOTOS_EXPORTDB_VERSION
         return conn
 
     def _get_db_connection(self, dbfile):
@@ -570,11 +623,20 @@ class ExportDB(ExportDB_ABC):
                               size INTEGER,
                               mtime REAL
                               ); """,
+            "sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar (
+                              id INTEGER PRIMARY KEY,
+                              filepath_normalized TEXT NOT NULL,
+                              sidecar_data TEXT,
+                              mode INTEGER,
+                              size INTEGER,
+                              mtime REAL
+                              ); """,
             "sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
             "sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
             "sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
             "sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
             "sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
+            "sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
         }
         try:
             c = conn.cursor()

osxphotos/photoinfo/_photoinfo_export.py

@@ -13,6 +13,7 @@
 # TODO: should this be its own PhotoExporter class?
 
 import glob
+import hashlib
 import json
 import logging
 import os
@@ -41,14 +42,31 @@ from ..photokit import (
     PhotoLibrary,
     PhotoKitFetchFailed,
 )
-from ..utils import dd_to_dms_str, findfiles
+from ..utils import dd_to_dms_str, findfiles, noop
 
 ExportResults = namedtuple(
     "ExportResults",
-    ["exported", "new", "updated", "skipped", "exif_updated", "touched"],
+    [
+        "exported",
+        "new",
+        "updated",
+        "skipped",
+        "exif_updated",
+        "touched",
+        "sidecar_json",
+        "sidecar_xmp",
+    ],
 )
 
 
+# hexdigest is not a class method, don't import this into PhotoInfo
+def hexdigest(strval):
+    """ hexdigest of a string, using blake2b """
+    h = hashlib.blake2b(digest_size=20)
+    h.update(bytes(strval, "utf-8"))
+    return h.hexdigest()
+
+
 # _export_photo_uuid_applescript is not a class method, don't import this into PhotoInfo
 def _export_photo_uuid_applescript(
     uuid,
@@ -149,11 +167,9 @@ def _export_photo_uuid_applescript(
                 and path.suffix.lower() == ".mov"
             ):
                 # it's the .mov part of live photo but not requested, so don't export
-                logging.debug(f"Skipping live photo file {path}")
                 continue
             if len(exported_files) > 1 and burst and path.stem != filename_stem:
                 # skip any burst photo that's not the one we asked for
-                logging.debug(f"Skipping burst photo file {path}")
                 continue
             if filestem:
                 # rename the file based on filestem, keeping original extension
@@ -161,7 +177,6 @@ def _export_photo_uuid_applescript(
             else:
                 # use the name Photos provided
                 dest_new = dest / path.name
-            logging.debug(f"exporting {path} to dest_new: {dest_new}")
             if not dry_run:
                 FileUtil.copy(str(path), str(dest_new))
             exported_paths.append(str(dest_new))
@@ -324,6 +339,7 @@ def export2(
     jpeg_quality=1.0,
     ignore_date_modified=False,
     use_photokit=False,
+    verbose=None,
 ):
     """ export photo, like export but with update and dry_run options
         dest: must be valid destination path or exception raised 
@@ -367,6 +383,7 @@ def export2(
         convert_to_jpeg: boolean; if True, converts non-jpeg images to jpeg
         jpeg_quality: float in range 0.0 <= jpeg_quality <= 1.0.  A value of 1.0 specifies use best quality, a value of 0.0 specifies use maximum compression.
         ignore_date_modified: for use with sidecar and exiftool; if True, sets EXIF:ModifyDate to EXIF:DateTimeOriginal even if date_modified is set
+        verbose: optional callable function to use for printing verbose text during processing; if None (default), does not print output.
 
         Returns: ExportResults namedtuple with fields: exported, new, updated, skipped 
                     where each field is a list of file paths
@@ -383,6 +400,12 @@ def export2(
     if export_db is None:
         export_db = ExportDBNoOp()
 
+    if verbose is None:
+        verbose = noop
+    elif not callable(verbose):
+        raise TypeError("verbose must be callable")
+    self._verbose = verbose
+
     # suffix to add to edited files
     # e.g. name will be filename_edited.jpg
     edited_identifier = "_edited"
@@ -488,11 +511,6 @@ def export2(
                     f"Cannot export edited photo if path_edited is None"
                 )
         else:
-            if self.ismissing:
-                logging.debug(
-                    f"Attempting to export photo with ismissing=True: path = {self.path}"
-                )
-
             if self.path is not None:
                 src = self.path
             else:
@@ -501,32 +519,22 @@ def export2(
         if not os.path.isfile(src):
             raise FileNotFoundError(f"{src} does not appear to exist")
 
-        if not _check_export_suffix(src, dest, edited):
-            logging.debug(
-                f"Invalid destination suffix: {dest.suffix} for {self.path}, "
-                + f"edited={edited}, path_edited={self.path_edited}, "
-                + f"original_filename={self.original_filename}, filename={self.filename}"
-            )
-
         # found source now try to find right destination
         if update and dest.exists():
             # destination exists, check to see if destination is the right UUID
             dest_uuid = export_db.get_uuid_for_file(dest)
             if dest_uuid is None and fileutil.cmp(src, dest):
                 # might be exporting into a pre-ExportDB folder or the DB got deleted
-                logging.debug(
-                    f"Found matching file with blank uuid: {self.uuid}, {dest}"
-                )
                 dest_uuid = self.uuid
                 export_db.set_data(
-                    dest,
-                    self.uuid,
-                    fileutil.file_sig(dest),
-                    (None, None, None),
-                    (None, None, None),
-                    (None, None, None),
-                    self.json(),
-                    None,
+                    filename=dest,
+                    uuid=self.uuid,
+                    orig_stat=fileutil.file_sig(dest),
+                    exif_stat=(None, None, None),
+                    converted_stat=(None, None, None),
+                    edited_stat=(None, None, None),
+                    info_json=self.json(),
+                    exif_json=None,
                 )
             if dest_uuid != self.uuid:
                 # not the right file, find the right one
@@ -545,14 +553,14 @@ def export2(
                         dest = pathlib.Path(file_)
                         found_match = True
                         export_db.set_data(
-                            dest,
-                            self.uuid,
-                            fileutil.file_sig(dest),
-                            (None, None, None),
-                            (None, None, None),
-                            (None, None, None),
-                            self.json(),
-                            None,
+                            filename=dest,
+                            uuid=self.uuid,
+                            orig_stat=fileutil.file_sig(dest),
+                            exif_stat=(None, None, None),
+                            converted_stat=(None, None, None),
+                            edited_stat=(None, None, None),
+                            info_json=self.json(),
+                            exif_json=None,
                         )
                         break
 
@@ -596,9 +604,6 @@ def export2(
             src_live = self.path_live_photo
 
             if src_live is not None:
-                logging.debug(
-                    f"Exporting live photo video of {filename} as {live_name.name}"
-                )
                 results = self._export_photo(
                     src_live,
                     live_name,
@@ -617,8 +622,6 @@ def export2(
                 update_updated_files.extend(results.updated)
                 update_skipped_files.extend(results.skipped)
                 touched_files.extend(results.touched)
-            else:
-                logging.debug(f"Skipping missing live movie for {filename}")
 
         # copy associated RAW image if requested
         if raw_photo and self.has_raw:
@@ -626,7 +629,6 @@ def export2(
             raw_ext = raw_path.suffix
             raw_name = dest.parent / f"{dest.stem}{raw_ext}"
             if raw_path is not None:
-                logging.debug(f"Exporting RAW photo of {filename} as {raw_name.name}")
                 results = self._export_photo(
                     raw_path,
                     raw_name,
@@ -646,8 +648,6 @@ def export2(
                 update_updated_files.extend(results.updated)
                 update_skipped_files.extend(results.skipped)
                 touched_files.extend(results.touched)
-            else:
-                logging.debug(f"Skipping missing RAW photo for {filename}")
     else:
         # use_photo_export
         exported = []
@@ -748,9 +748,10 @@ def export2(
             )
 
     # export metadata
+    sidecar_json_files = []
     if sidecar_json:
-        logging.debug("writing exiftool_json_sidecar")
         sidecar_filename = dest.parent / pathlib.Path(f"{dest.stem}{dest.suffix}.json")
+        sidecar_json_files.append(str(sidecar_filename))
         sidecar_str = self._exiftool_json_sidecar(
             use_albums_as_keywords=use_albums_as_keywords,
             use_persons_as_keywords=use_persons_as_keywords,
@@ -758,16 +759,35 @@ def export2(
             description_template=description_template,
             ignore_date_modified=ignore_date_modified,
         )
-        if not dry_run:
-            try:
+        sidecar_digest = hexdigest(sidecar_str)
+        old_sidecar_digest, sidecar_sig = export_db.get_sidecar_for_file(
+            sidecar_filename
+        )
+        write_sidecar = (
+            not update
+            or (update and not sidecar_filename.exists())
+            or (
+                update
+                and (sidecar_digest != old_sidecar_digest)
+                or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig)
+            )
+        )
+        if write_sidecar:
+            verbose(f"Writing exiftool JSON sidecar {sidecar_filename}")
+            if not dry_run:
                 self._write_sidecar(sidecar_filename, sidecar_str)
-            except Exception as e:
-                logging.warning(f"Error writing json sidecar to {sidecar_filename}")
-                raise e
+                export_db.set_sidecar_for_file(
+                    sidecar_filename,
+                    sidecar_digest,
+                    fileutil.file_sig(sidecar_filename),
+                )
+        else:
+            verbose(f"Skipped up to date exiftool JSON sidecar {sidecar_filename}")
 
+    sidecar_xmp_files = []
     if sidecar_xmp:
-        logging.debug("writing xmp_sidecar")
         sidecar_filename = dest.parent / pathlib.Path(f"{dest.stem}{dest.suffix}.xmp")
+        sidecar_xmp_files.append(str(sidecar_filename))
         sidecar_str = self._xmp_sidecar(
             use_albums_as_keywords=use_albums_as_keywords,
             use_persons_as_keywords=use_persons_as_keywords,
@@ -775,12 +795,30 @@ def export2(
             description_template=description_template,
             extension=dest.suffix[1:] if dest.suffix else None,
         )
-        if not dry_run:
-            try:
+        sidecar_digest = hexdigest(sidecar_str)
+        old_sidecar_digest, sidecar_sig = export_db.get_sidecar_for_file(
+            sidecar_filename
+        )
+        write_sidecar = (
+            not update
+            or (update and not sidecar_filename.exists())
+            or (
+                update
+                and (sidecar_digest != old_sidecar_digest)
+                or not fileutil.cmp_file_sig(sidecar_filename, sidecar_sig)
+            )
+        )
+        if write_sidecar:
+            verbose(f"Writing XMP sidecar {sidecar_filename}")
+            if not dry_run:
                 self._write_sidecar(sidecar_filename, sidecar_str)
-            except Exception as e:
-                logging.warning(f"Error writing xmp sidecar to {sidecar_filename}")
-                raise e
+                export_db.set_sidecar_for_file(
+                    sidecar_filename,
+                    sidecar_digest,
+                    fileutil.file_sig(sidecar_filename),
+                )
+        else:
+            verbose(f"Skipped up to date XMP sidecar {sidecar_filename}")
 
     # if exiftool, write the metadata
     if update:
@@ -791,7 +829,6 @@ def export2(
     exif_files_updated = []
     if exiftool and update and exif_files:
         for exported_file in exif_files:
-            logging.debug(f"checking exif for {exported_file}")
             files_are_different = False
             old_data = export_db.get_exifdata_for_file(exported_file)
             if old_data is not None:
@@ -811,6 +848,7 @@ def export2(
             if old_data is None or files_are_different:
                 # didn't have old data, assume we need to write it
                 # or files were different
+                verbose(f"Writing metadata with exiftool for {exported_file}")
                 if not dry_run:
                     self._write_exif_data(
                         exported_file,
@@ -834,8 +872,11 @@ def export2(
                     exported_file, fileutil.file_sig(exported_file)
                 )
                 exif_files_updated.append(exported_file)
+            else:
+                verbose(f"Skipped up to date exiftool metadata for {exported_file}")
     elif exiftool and exif_files:
         for exported_file in exif_files:
+            verbose(f"Writing metadata with exiftool for {exported_file}")
             if not dry_run:
                 self._write_exif_data(
                     exported_file,
@@ -863,6 +904,7 @@ def export2(
 
     if touch_file:
         for exif_file in exif_files_updated:
+            verbose(f"Updating file modification time for {exif_file}")
             touched_files.append(exif_file)
             ts = int(self.date.timestamp())
             fileutil.utime(exif_file, (ts, ts))
@@ -876,6 +918,8 @@ def export2(
         update_skipped_files,
         exif_files_updated,
         touched_files,
+        sidecar_json_files,
+        sidecar_xmp_files,
     )
     return results
 
@@ -998,13 +1042,11 @@ def _export_photo(
 
         else:
             # update, destination doesn't exist (new file)
-            logging.debug(f"Update: exporting new file with {op_desc} {src} {dest}")
             update_new_files.append(dest_str)
             if touch_file:
                 touched_files.append(dest_str)
     else:
         # not update, export the file
-        logging.debug(f"Exporting file with {op_desc} {src} {dest}")
         exported_files.append(dest_str)
         if touch_file:
             sig = fileutil.file_sig(src)
@@ -1016,9 +1058,6 @@ def _export_photo(
         edited_stat = fileutil.file_sig(src) if edited else (None, None, None)
         if dest_exists and (update or overwrite):
             # need to remove the destination first
-            logging.debug(
-                f"Update: removing existing file prior to {op_desc} {src} {dest}"
-            )
             fileutil.unlink(dest)
         if export_as_hardlink:
             fileutil.hardlink(src, dest)
@@ -1030,14 +1069,14 @@ def _export_photo(
             fileutil.copy(src, dest_str, norsrc=no_xattr)
 
         export_db.set_data(
-            dest_str,
-            self.uuid,
-            fileutil.file_sig(dest_str),
-            (None, None, None),
-            converted_stat,
-            edited_stat,
-            self.json(),
-            None,
+            filename=dest_str,
+            uuid=self.uuid,
+            orig_stat=fileutil.file_sig(dest_str),
+            exif_stat=(None, None, None),
+            converted_stat=converted_stat,
+            edited_stat=edited_stat,
+            info_json=self.json(),
+            exif_json=None,
         )
 
     if touched_files:
@@ -1051,6 +1090,8 @@ def _export_photo(
         update_skipped_files,
         [],
         touched_files,
+        [],
+        [],
     )
 
 
@@ -1121,9 +1162,9 @@ def _exiftool_dict(
         IPTC:Keywords (may include album name, person name, or template)
         XMP:Subject
         XMP:PersonInImage
+        EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
         EXIF:GPSLatitude, EXIF:GPSLongitude
         EXIF:GPSPosition
-        EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
         EXIF:DateTimeOriginal
         EXIF:OffsetTimeOriginal
         EXIF:ModifyDate
@@ -1283,9 +1324,9 @@ def _exiftool_json_sidecar(
         IPTC:Keywords (may include album name, person name, or template)
         XMP:Subject
         XMP:PersonInImage
+        EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
         EXIF:GPSLatitude, EXIF:GPSLongitude
         EXIF:GPSPosition
-        EXIF:GPSLatitudeRef, EXIF:GPSLongitudeRef
         EXIF:DateTimeOriginal
         EXIF:OffsetTimeOriginal
         EXIF:ModifyDate

tests/test_cli.py

@@ -976,7 +976,9 @@ def test_export_exiftool_ignore_date_modified():
             )
             assert result.exit_code == 0
 
-            exif = ExifTool(CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid]["File:FileName"]).asdict()
+            exif = ExifTool(
+                CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid]["File:FileName"]
+            ).asdict()
             for key in CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid]:
                 assert exif[key] == CLI_EXIFTOOL_IGNORE_DATE_MODIFIED[uuid][key]
 
@@ -1734,6 +1736,142 @@ def test_export_sidecar_templates():
         )
 
 
+def test_export_sidecar_update():
+    """ test sidecar don't update if not changed and do update if changed """
+    import datetime
+    import glob
+    import os
+    import os.path
+    import osxphotos
+    from osxphotos.fileutil import FileUtil
+
+    from osxphotos.__main__ import cli
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Writing XMP sidecar" in result.output
+        assert "Writing exiftool JSON sidecar" in result.output
+
+        # delete a sidecar file and run update
+        fileutil = FileUtil()
+        fileutil.unlink(CLI_EXPORT_SIDECAR_FILENAMES[1])
+
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+                "--update",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Skipped up to date XMP sidecar" in result.output
+        assert "Writing exiftool JSON sidecar" in result.output
+
+        # run update again, no sidecar files should update
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+                "--update",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Skipped up to date XMP sidecar" in result.output
+        assert "Skipped up to date exiftool JSON sidecar" in result.output
+
+        # touch a file and export again
+        ts = datetime.datetime.now().timestamp() + 1000
+        fileutil.utime(CLI_EXPORT_SIDECAR_FILENAMES[2], (ts, ts))
+
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+                "--update",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Writing XMP sidecar" in result.output
+        assert "Skipped up to date exiftool JSON sidecar" in result.output
+
+        # run update again, no sidecar files should update
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+                "--update",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Skipped up to date XMP sidecar" in result.output
+        assert "Skipped up to date exiftool JSON sidecar" in result.output
+
+        # run update again with updated metadata, forcing update
+        result = runner.invoke(
+            cli,
+            [
+                "export",
+                "--db",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "--sidecar=json",
+                "--sidecar=xmp",
+                f"--uuid={CLI_EXPORT_UUID}",
+                "-V",
+                "--update",
+                "--keyword-template",
+                "foo",
+            ],
+        )
+        assert result.exit_code == 0
+        assert "Writing XMP sidecar" in result.output
+        assert "Writing exiftool JSON sidecar" in result.output
+
+
 def test_export_live():
     import glob
     import os

tests/test_export_db.py

@@ -4,6 +4,8 @@ import pytest
 
 EXIF_DATA = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "EXIF:ImageDescription": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "XMP:Title": "Elder Park", "EXIF:GPSLatitude": "34 deg 55' 8.01\" S", "EXIF:GPSLongitude": "138 deg 35' 48.70\" E", "Composite:GPSPosition": "34 deg 55' 8.01\" S, 138 deg 35' 48.70\" E", "EXIF:GPSLatitudeRef": "South", "EXIF:GPSLongitudeRef": "East", "EXIF:DateTimeOriginal": "2017:06:20 17:18:56", "EXIF:OffsetTimeOriginal": "+09:30", "EXIF:ModifyDate": "2020:05:18 14:42:04"}]"""
 INFO_DATA = """{"uuid": "3DD2C897-F19E-4CA6-8C22-B027D5A71907", "filename": "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "original_filename": "IMG_4547.jpg", "date": "2017-06-20T17:18:56.518000+09:30", "description": "\u2068Elder Park\u2069, \u2068Adelaide\u2069, \u2068Australia\u2069", "title": "Elder Park", "keywords": [], "labels": ["Statue", "Art"], "albums": ["AlbumInFolder"], "folders": {"AlbumInFolder": ["Folder1", "SubFolder2"]}, "persons": [], "path": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/originals/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg", "ismissing": false, "hasadjustments": true, "external_edit": false, "favorite": false, "hidden": false, "latitude": -34.91889167000001, "longitude": 138.59686167, "path_edited": "/Users/rhet/Pictures/Test-10.15.4.photoslibrary/resources/renders/3/3DD2C897-F19E-4CA6-8C22-B027D5A71907_1_201_a.jpeg", "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": false, "incloud": null, "date_modified": "2020-05-18T14:42:04.608664+09:30", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Elder Park, Adelaide, South Australia, Australia, River Torrens", "names": {"field0": [], "country": ["Australia"], "state_province": ["South Australia"], "sub_administrative_area": ["Adelaide"], "city": ["Adelaide", "Adelaide"], "field5": [], "additional_city_info": ["Adelaide CBD", "Tarndanya"], "ocean": [], "area_of_interest": ["Elder Park", ""], "inland_water": ["River Torrens", "River Torrens"], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": ["River Torrens", "River Torrens"]}, "country_code": "AU", "ishome": false, "address_str": "River Torrens, Adelaide SA, Australia", "address": {"street": null, "sub_locality": "Tarndanya", "city": "Adelaide", "sub_administrative_area": "Adelaide", "state_province": "SA", "postal_code": null, "country": "Australia", "iso_country_code": "AU"}}, "exif": {"flash_fired": false, "iso": 320, "metering_mode": 3, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.058823529411764705, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}"""
+SIDECAR_DATA = """FOO_BAR"""
+
 EXIF_DATA2 = """[{"_CreatedBy": "osxphotos, https://github.com/RhetTbull/osxphotos", "XMP:Title": "St. James's Park", "XMP:TagsList": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "IPTC:Keywords": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "XMP:Subject": ["London 2018", "St. James's Park", "England", "United Kingdom", "UK", "London"], "EXIF:GPSLatitude": "51 deg 30' 12.86\" N", "EXIF:GPSLongitude": "0 deg 7' 54.50\" W", "Composite:GPSPosition": "51 deg 30' 12.86\" N, 0 deg 7' 54.50\" W", "EXIF:GPSLatitudeRef": "North", "EXIF:GPSLongitudeRef": "West", "EXIF:DateTimeOriginal": "2018:10:13 09:18:12", "EXIF:OffsetTimeOriginal": "-04:00", "EXIF:ModifyDate": "2019:12:08 14:06:44"}]"""
 INFO_DATA2 = """{"uuid": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529", "filename": "F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "original_filename": "IMG_8440.JPG", "date": "2019-06-11T11:42:06.711805-07:00", "description": null, "title": null, "keywords": [], "labels": ["Sky", "Cloudy", "Fence", "Land", "Outdoor", "Park", "Amusement Park", "Roller Coaster"], "albums": [], "folders": {}, "persons": [], "path": "/Volumes/MacBook Catalina - Data/Users/rhet/Pictures/Photos Library.photoslibrary/originals/F/F2BB3F98-90F0-4E4C-A09B-25C6822A4529.jpeg", "ismissing": false, "hasadjustments": false, "external_edit": false, "favorite": false, "hidden": false, "latitude": 33.81558666666667, "longitude": -117.99298, "path_edited": null, "shared": false, "isphoto": true, "ismovie": false, "uti": "public.jpeg", "burst": false, "live_photo": false, "path_live_photo": null, "iscloudasset": true, "incloud": true, "date_modified": "2019-10-14T00:51:47.141950-07:00", "portrait": false, "screenshot": false, "slow_mo": false, "time_lapse": false, "hdr": false, "selfie": false, "panorama": false, "has_raw": false, "uti_raw": null, "path_raw": null, "place": {"name": "Adventure City, Stanton, California, United States", "names": {"field0": [], "country": ["United States"], "state_province": ["California"], "sub_administrative_area": ["Orange"], "city": ["Stanton", "Anaheim", "Anaheim"], "field5": [], "additional_city_info": ["West Anaheim"], "ocean": [], "area_of_interest": ["Adventure City", "Adventure City"], "inland_water": [], "field10": [], "region": [], "sub_throughfare": [], "field13": [], "postal_code": [], "field15": [], "field16": [], "street_address": [], "body_of_water": []}, "country_code": "US", "ishome": false, "address_str": "Adventure City, 1240 S Beach Blvd, Anaheim, CA  92804, United States", "address": {"street": "1240 S Beach Blvd", "sub_locality": "West Anaheim", "city": "Stanton", "sub_administrative_area": "Orange", "state_province": "CA", "postal_code": "92804", "country": "United States", "iso_country_code": "US"}}, "exif": {"flash_fired": false, "iso": 25, "metering_mode": 5, "sample_rate": null, "track_format": null, "white_balance": 0, "aperture": 2.2, "bit_rate": null, "duration": null, "exposure_bias": 0.0, "focal_length": 4.15, "fps": null, "latitude": null, "longitude": null, "shutter_speed": 0.0004940711462450593, "camera_make": "Apple", "camera_model": "iPhone 6s", "codec": null, "lens_model": "iPhone 6s back camera 4.15mm f/2.2"}}"""
 DATABASE_VERSION1 = "tests/export_db_version1.db"
@@ -41,6 +43,8 @@ def test_export_db():
     assert db.get_stat_edited_for_file(filepath) == (10, 11, 12)
     db.set_stat_converted_for_file(filepath, (7, 8, 9))
     assert db.get_stat_converted_for_file(filepath) == (7, 8, 9)
+    db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15))
+    assert db.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15))
 
     # test set_data which sets all at the same time
     filepath2 = os.path.join(tempdir.name, "test2.jpg")
@@ -109,6 +113,8 @@ def test_export_db_no_op():
     assert db.get_stat_converted_for_file(filepath) is None
     db.set_stat_edited_for_file(filepath, (10, 11, 12))
     assert db.get_stat_edited_for_file(filepath) is None
+    db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15))
+    assert db.get_sidecar_for_file(filepath) == (None, (None, None, None))
 
     # test set_data which sets all at the same time
     filepath2 = os.path.join(tempdir.name, "test2.jpg")
@@ -160,6 +166,7 @@ def test_export_db_in_memory():
     db.set_stat_exif_for_file(filepath, (4, 5, 6))
     db.set_stat_converted_for_file(filepath, (7, 8, 9))
     db.set_stat_edited_for_file(filepath, (10, 11, 12))
+    db.set_sidecar_for_file(filepath, SIDECAR_DATA, (13, 14, 15))
 
     db.close()
 
@@ -176,6 +183,7 @@ def test_export_db_in_memory():
     assert dbram.get_stat_exif_for_file(filepath) == (4, 5, 6)
     assert dbram.get_stat_converted_for_file(filepath) == (7, 8, 9)
     assert dbram.get_stat_edited_for_file(filepath) == (10, 11, 12)
+    assert dbram.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15))
 
     # change a value
     dbram.set_uuid_for_file(filepath, "FUBAR")
@@ -185,6 +193,7 @@ def test_export_db_in_memory():
     dbram.set_stat_exif_for_file(filepath, (10, 11, 12))
     dbram.set_stat_converted_for_file(filepath, (1, 2, 3))
     dbram.set_stat_edited_for_file(filepath, (4, 5, 6))
+    dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22))
 
     assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR"
     assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2
@@ -193,6 +202,7 @@ def test_export_db_in_memory():
     assert dbram.get_stat_exif_for_file(filepath) == (10, 11, 12)
     assert dbram.get_stat_converted_for_file(filepath) == (1, 2, 3)
     assert dbram.get_stat_edited_for_file(filepath) == (4, 5, 6)
+    assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22))
 
     dbram.close()
 
@@ -205,6 +215,7 @@ def test_export_db_in_memory():
     assert db.get_stat_exif_for_file(filepath) == (4, 5, 6)
     assert db.get_stat_converted_for_file(filepath) == (7, 8, 9)
     assert db.get_stat_edited_for_file(filepath) == (10, 11, 12)
+    assert db.get_sidecar_for_file(filepath) == (SIDECAR_DATA, (13, 14, 15))
 
     assert db.get_info_for_uuid("FUBAR") is None
 
@@ -232,6 +243,7 @@ def test_export_db_in_memory_nofile():
     dbram.set_stat_exif_for_file(filepath, (10, 11, 12))
     dbram.set_stat_converted_for_file(filepath, (1, 2, 3))
     dbram.set_stat_edited_for_file(filepath, (4, 5, 6))
+    dbram.set_sidecar_for_file(filepath, "FUBAR", (20, 21, 22))
 
     assert dbram.get_uuid_for_file(filepath_lower) == "FUBAR"
     assert dbram.get_info_for_uuid("FUBAR") == INFO_DATA2
@@ -240,5 +252,6 @@ def test_export_db_in_memory_nofile():
     assert dbram.get_stat_exif_for_file(filepath) == (10, 11, 12)
     assert dbram.get_stat_converted_for_file(filepath) == (1, 2, 3)
     assert dbram.get_stat_edited_for_file(filepath) == (4, 5, 6)
+    assert dbram.get_sidecar_for_file(filepath) == ("FUBAR", (20, 21, 22))
 
     dbram.close()