Added download-missing option to CLI export

README.md

@@ -648,7 +648,7 @@ Returns the path to the live video component of a [live photo](#live_photo). If
 #### `json()`
 Returns a JSON representation of all photo info 
 
-#### `export(dest, *filename, edited=False, overwrite=False, increment=True, sidecar=False)`
+#### `export(dest, *filename, edited=False, overwrite=False, increment=True, sidecar=False, use_photos_export=False, timeout=120)`
 
 Export photo from the Photos library to another destination on disk.  
 - dest: must be valid destination path as str (or exception raised).
@@ -656,7 +656,9 @@ Export photo from the Photos library to another destination on disk.
 - edited: boolean; if True (default=False), will export the edited version of the photo (or raise exception if no edited version)
 - overwrite: boolean; if True (default=False), will overwrite files if they alreay exist
 - increment: boolean; if True (default=True), will increment file name until a non-existant name is found
-- sidecar: boolean; if True (default=False) will also write a json sidecar file with EXIF data in format readable by [exiftool](https://exiftool.org/); filename will be dest/filename.ext.json where ext is suffix of the image file (e.g. jpeg or jpg)
+- sidecar: boolean; if True (default=False) will also write a json sidecar file with EXIF data in format readable by [exiftool](https://exiftool.org/); filename will be dest/filename.ext.json where ext is suffix of the image file (e.g. jpeg or jpg). Note: this is not an XMP sidecar.
+- use_photos_export: boolean; (default=False), if True will attempt to export photo via applescript interaction with Photos--useful for forcing download of missing photos
+- timeout: (int, default=120) timeout in seconds used with use_photos_export
 
 The json sidecar file can be used by exiftool to apply the metadata from the json file to the image.  For example: 
 

osxphotos/__main__.py

@@ -266,10 +266,26 @@ def list_libraries(cli_obj):
 @click.option(
     "--not-live", is_flag=True, help="Search for photos that are not Apple live photos"
 )
-@click.option("--cloudasset", is_flag=True, help="Search for photos that are part of an iCloud library")
+@click.option(
-@click.option("--not-cloudasset", is_flag=True, help="Search for photos that are not part of an iCloud library")
+    "--cloudasset",
-@click.option("--incloud", is_flag=True, help="Search for photos that are in iCloud (have been synched)")
+    is_flag=True,
-@click.option("--not-incloud", is_flag=True, help="Search for photos that are not in iCloud (have not been synched)")
+    help="Search for photos that are part of an iCloud library",
+)
+@click.option(
+    "--not-cloudasset",
+    is_flag=True,
+    help="Search for photos that are not part of an iCloud library",
+)
+@click.option(
+    "--incloud",
+    is_flag=True,
+    help="Search for photos that are in iCloud (have been synched)",
+)
+@click.option(
+    "--not-incloud",
+    is_flag=True,
+    help="Search for photos that are not in iCloud (have not been synched)",
+)
 @click.option(
     "--only-movies",
     is_flag=True,
@@ -445,7 +461,7 @@ def query(
         cloudasset,
         not_cloudasset,
         incloud,
-        not_incloud
+        not_incloud,
     )
     print_photo_info(photos, cli_obj.json or json)
 
@@ -550,11 +566,12 @@ def query(
 @click.option(
     "--sidecar",
     is_flag=True,
-    help="Create json sidecar for each photo exported "
+    help="Create JSON sidecar for each photo exported "
     f"in format useable by exiftool ({_EXIF_TOOL_URL}) "
     "The sidecar file can be used to apply metadata to the file with exiftool, for example: "
     '"exiftool -j=photoname.jpg.json photoname.jpg" '
-    "The sidecar file is named in format photoname.ext.json where ext is extension of the photo (e.g. jpg).",
+    "The sidecar file is named in format photoname.ext.json where ext is extension of the photo (e.g. jpg). "
+    "Note: this does not create an XMP sidecar as used by Lightroom, etc.",
 )
 @click.option(
     "--only-movies",
@@ -566,6 +583,14 @@ def query(
     is_flag=True,
     help="Search only for photos/images (default searches both images and movies).",
 )
+@click.option(
+    "--download-missing",
+    is_flag=True,
+    help="Attempt to download missing photos from iCloud. The current implementation uses Applescript "
+    "to interact with Photos to export the photo which will force Photos to download from iCloud if "
+    "the photo does not exist on disk.  This will be slow and will require internet connection. "
+    "This obviously only works if the Photos library is synched to iCloud.",
+)
 @click.argument("dest", nargs=1)
 @click.pass_obj
 @click.pass_context
@@ -604,6 +629,7 @@ def export(
     not_burst,
     live,
     not_live,
+    download_missing,
     dest,
 ):
     """ Export photos from the Photos database.
@@ -682,6 +708,10 @@ def export(
         not_burst,
         live,
         not_live,
+        False, # cloudasset
+        False, # not_cloudasset
+        False, # incloud
+        False # not_incloud
     )
 
     if photos:
@@ -709,6 +739,7 @@ def export(
                         export_edited,
                         original_name,
                         export_live,
+                        download_missing
                     )
         else:
             for p in photos:
@@ -722,6 +753,7 @@ def export(
                     export_edited,
                     original_name,
                     export_live,
+                    download_missing
                 )
                 if export_path:
                     click.echo(f"Exported {p.filename} to {export_path}")
@@ -951,12 +983,12 @@ def _query(
         photos = [p for p in photos if p.iscloudasset]
     elif not_cloudasset:
         photos = [p for p in photos if not p.iscloudasset]
-    
+
     if incloud:
         photos = [p for p in photos if p.incloud]
     elif not_incloud:
         photos = [p for p in photos if not p.incloud]
-        
+
     return photos
 
 
@@ -970,6 +1002,7 @@ def export_photo(
     export_edited,
     original_name,
     export_live,
+    download_missing,
 ):
     """ Helper function for export that does the actual export
         photo: PhotoInfo object
@@ -981,19 +1014,24 @@ def export_photo(
         original_name: boolean; use original filename instead of current filename
         export_live: boolean; also export live video component if photo is a live photo
                      live video will have same name as photo but with .mov extension
+        download_missing: attempt download of missing iCloud photos
         returns destination path of exported photo or None if photo was missing 
     """
 
-    if photo.ismissing:
+    if not download_missing:
-        space = " " if not verbose else ""
+        if photo.ismissing:
-        click.echo(f"{space}Skipping missing photo {photo.filename}")
+            space = " " if not verbose else ""
-        return None
+            click.echo(f"{space}Skipping missing photo {photo.filename}")
-    elif not os.path.exists(photo.path):
+            return None
-        space = " " if not verbose else ""
+        elif not os.path.exists(photo.path):
-        click.echo(
+            space = " " if not verbose else ""
-            f"{space}WARNING: file {photo.path} is missing but ismissing=False, "
+            click.echo(
-            f"skipping {photo.filename}"
+                f"{space}WARNING: file {photo.path} is missing but ismissing=False, "
-        )
+                f"skipping {photo.filename}"
+            )
+            return None
+    elif photo.ismissing and not photo.iscloudasset or not photo.incloud:
+        click.echo(f"Skipping missing {photo.filename}: not iCloud asset or missing from cloud")
         return None
 
     filename = None
@@ -1009,18 +1047,32 @@ def export_photo(
         date_created = photo.date.timetuple()
         dest = create_path_by_date(dest, date_created)
 
-    photo_path = photo.export(dest, filename, sidecar=sidecar, overwrite=overwrite)
+    photo_path = photo.export(
+        dest,
+        filename,
+        sidecar=sidecar,
+        overwrite=overwrite,
+        use_photos_export=download_missing,
+    )
 
     # if export-edited, also export the edited version
     # verify the photo has adjustments and valid path to avoid raising an exception
-    if export_edited and photo.hasadjustments and photo.path_edited is not None:
+    if export_edited and photo.hasadjustments:
-        edited_name = pathlib.Path(filename)
+        if download_missing or photo.path_edited is not None:
-        edited_name = f"{edited_name.stem}_edited{edited_name.suffix}"
+            edited_name = pathlib.Path(filename)
-        if verbose:
+            edited_name = f"{edited_name.stem}_edited{edited_name.suffix}"
-            click.echo(f"Exporting edited version of {filename} as {edited_name}")
+            if verbose:
-        photo.export(
+                click.echo(f"Exporting edited version of {filename} as {edited_name}")
-            dest, edited_name, sidecar=sidecar, overwrite=overwrite, edited=True
+            photo.export(
-        )
+                dest,
+                edited_name,
+                sidecar=sidecar,
+                overwrite=overwrite,
+                edited=True,
+                use_photos_export=download_missing,
+            )
+        else:
+            click.echo(f"Skipping missing edited photo for {filename}")
 
     if export_live and photo.live_photo and photo.path_live_photo is not None:
         # if destination exists, will be overwritten regardless of overwrite
@@ -1031,11 +1083,14 @@ def export_photo(
         src_live = photo.path_live_photo
         dest_live = pathlib.Path(photo_path).parent / pathlib.Path(live_name)
 
-        if verbose:
+        if src_live is not None:
-            click.echo(f"Exporting live photo video of {filename} as {live_name}")
+            if verbose:
-        
+                click.echo(f"Exporting live photo video of {filename} as {live_name}")
-        _copy_file(src_live, str(dest_live))
+
-        
+            _copy_file(src_live, str(dest_live))
+        else:
+           click.echo(f"Skipping missing live movie for {filename}")
+
     return photo_path
 
 

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.21.4"
+__version__ = "0.21.5"

osxphotos/photoinfo.py

@@ -16,9 +16,18 @@ from pprint import pformat
 
 import yaml
 
-from ._constants import (_MOVIE_TYPE, _PHOTO_TYPE, _PHOTOS_5_SHARED_PHOTO_PATH,
+from ._constants import (
-                         _PHOTOS_5_VERSION)
+    _MOVIE_TYPE,
-from .utils import _copy_file, _get_resource_loc, dd_to_dms_str
+    _PHOTO_TYPE,
+    _PHOTOS_5_SHARED_PHOTO_PATH,
+    _PHOTOS_5_VERSION,
+)
+from .utils import (
+    _copy_file,
+    _get_resource_loc,
+    dd_to_dms_str,
+    _export_photo_uuid_applescript,
+)
 
 # TODO: check pylint output
 
@@ -151,18 +160,15 @@ class PhotoInfo:
 
                     if not os.path.isfile(photopath):
                         rootdir = os.path.join(
-                        library,
+                            library, "resources", "media", "version", folder_id
-                        "resources",
+                        )
-                        "media",
+
-                        "version",
-                        folder_id)
-                        
                         for dirname, _, filelist in os.walk(rootdir):
                             if filename in filelist:
                                 photopath = os.path.join(dirname, filename)
                                 break
 
-                    # check again to see if we found a valid file    
+                    # check again to see if we found a valid file
                     if not os.path.isfile(photopath):
                         logging.warning(
                             f"MISSING PATH: edited file for UUID {self._uuid} should be at {photopath} but does not appear to exist"
@@ -492,37 +498,7 @@ class PhotoInfo:
                 else:
                     filename = self.filename
 
-        # get path to source file and verify it's not None and is valid file
+        # check destination path
-        # TODO: how to handle ismissing or not hasadjustments and edited=True cases?
-        if edited:
-            if not self.hasadjustments:
-                logging.warning(
-                    "Attempting to export edited photo but hasadjustments=False"
-                )
-
-            if self.path_edited is not None:
-                src = self.path_edited
-            else:
-                raise FileNotFoundError(
-                    f"edited=True but path_edited is none; hasadjustments: {self.hasadjustments}"
-                )
-        else:
-            if self.ismissing:
-                logging.warning(
-                    f"Attempting to export photo with ismissing=True: path = {self.path}"
-                )
-
-            if self.path is None:
-                logging.warning(
-                    f"Attempting to export photo but path is None: ismissing = {self.ismissing}"
-                )
-                raise FileNotFoundError("Cannot export photo if path is None")
-            else:
-                src = self.path
-
-        if not os.path.isfile(src):
-            raise FileNotFoundError(f"{src} does not appear to exist")
-
         dest = pathlib.Path(dest)
         filename = pathlib.Path(filename)
         dest = dest / filename
@@ -536,18 +512,67 @@ class PhotoInfo:
                 count += 1
             dest = dest_new
 
-        logging.debug(
-            f"exporting {src} to {dest}, overwrite={overwrite}, incremetn={increment}, dest exists: {dest.exists()}"
-        )
-
         # if overwrite==False and #increment==False, export should fail if file exists
         if dest.exists() and not overwrite and not increment:
             raise FileExistsError(
                 f"destination exists ({dest}); overwrite={overwrite}, increment={increment}"
             )
 
-        # copy the file, _copy_file uses ditto to preserve Mac extended attributes
+        if not use_photos_export:
-        _copy_file(src, dest)
+            # find the source file on disk and export
+            # get path to source file and verify it's not None and is valid file
+            # TODO: how to handle ismissing or not hasadjustments and edited=True cases?
+            if edited:
+                if not self.hasadjustments:
+                    logging.warning(
+                        "Attempting to export edited photo but hasadjustments=False"
+                    )
+
+                if self.path_edited is not None:
+                    src = self.path_edited
+                else:
+                    raise FileNotFoundError(
+                        f"edited=True but path_edited is none; hasadjustments: {self.hasadjustments}"
+                    )
+            else:
+                if self.ismissing:
+                    logging.warning(
+                        f"Attempting to export photo with ismissing=True: path = {self.path}"
+                    )
+
+                if self.path is None:
+                    logging.warning(
+                        f"Attempting to export photo but path is None: ismissing = {self.ismissing}"
+                    )
+                    raise FileNotFoundError("Cannot export photo if path is None")
+                else:
+                    src = self.path
+
+            if not os.path.isfile(src):
+                raise FileNotFoundError(f"{src} does not appear to exist")
+
+            logging.debug(
+                f"exporting {src} to {dest}, overwrite={overwrite}, increment={increment}, dest exists: {dest.exists()}"
+            )
+
+            # copy the file, _copy_file uses ditto to preserve Mac extended attributes
+            _copy_file(src, dest)
+        else:
+            # use_photo_export
+            exported = None
+            if edited:
+                # exported edited version and not original
+                exported = _export_photo_uuid_applescript(
+                    self.uuid, dest, original=False, edited=True, timeout=timeout
+                )
+            else:
+                # export original version and not edited
+                exported = _export_photo_uuid_applescript(
+                    self.uuid, dest, original=True, edited=False, timeout=timeout
+                )
+
+            if exported is None:
+                logging.warning(f"Error exporting photo {photo.uuid} to {dest}")
 
         if sidecar:
             logging.debug("writing exiftool_json_sidecar")

osxphotos/utils.py

@@ -300,6 +300,7 @@ def _export_photo_uuid_applescript(
         edited: (boolean) if True, export edited photo; default = False
                 will produce an error if image does not have edits/adjustments  
         timeout: timeout value in seconds; export will fail if applescript run time exceeds timeout
+        Returns: path to exported file or None if export failed
     """
 
     # setup the applescript to do the export
@@ -345,7 +346,10 @@ def _export_photo_uuid_applescript(
         return None
 
     if filename is not None:
-        path = os.path.join(tmpdir.name, filename)
+        # need to find actual filename as sometimes Photos renames JPG to jpeg on export
+        # this assumes only a single file in export folder, which should be true as
+        # TemporaryDirectory will cleanup on return
+        path = glob.glob(os.path.join(tmpdir.name, "*"))[0]
         _copy_file(path, dest)
         if os.path.isdir(dest):
             new_path = os.path.join(dest, filename)