Added support for movies for Photos 5; fixed bugs in ismissing and path

README.md

@@ -25,7 +25,7 @@
       - [`library_path`](#library_path)
       - [`db_path`](#db_path)
       - [`db_version`](#db_version)
-      - [`photos(keywords=[], uuid=[], persons=[], albums=[])`](#photoskeywords-uuid-persons-albums)
+      - [` photos(keywords=None, uuid=None, persons=None, albums=None, images=True, movies=False)`](#-photoskeywordsnone-uuidnone-personsnone-albumsnone-imagestrue-moviesfalse)
     + [PhotoInfo](#photoinfo)
       - [`uuid`](#uuid)
       - [`filename`](#filename)
@@ -45,6 +45,9 @@
       - [`hidden`](#hidden)
       - [`location`](#location)
       - [`shared`](#shared)
+      - [`isphoto`](#isphoto)
+      - [`ismovie`](#ismovie)
+      - [`uti`](#uti)
       - [`json()`](#json)
       - [`export(dest, *filename, edited=False, overwrite=False, increment=True, sidecar=False)`](#exportdest-filename-editedfalse-overwritefalse-incrementtrue-sidecarfalse)
     + [Utility Functions](#utility-functions)
@@ -380,7 +383,7 @@ photosdb.db_version
 Returns the version number for Photos library database.  You likely won't need this but it's provided in case needed for debugging. PhotosDB will print a warning to `sys.stderr` if you open a database version that has not been tested. 
 
 
-#### `photos(keywords=[], uuid=[], persons=[], albums=[])`
+#### ` photos(keywords=None, uuid=None, persons=None, albums=None, images=True, movies=False)`
 
 ```python
 # assumes photosdb is a PhotosDB object (see above)
@@ -397,7 +400,9 @@ photos = photosdb.photos(
     keywords = [],
     uuid = [],
     persons = [],
-    albums = []
+    albums = [],
+    images = bool,
+    movies = bool,
 )
 ```
 
@@ -405,8 +410,10 @@ photos = photosdb.photos(
 - ```uuid```: list of one or more uuids.  Returns only photos whos UUID matches.  **Note**: The UUID is the universally unique identifier that the Photos database uses to identify each photo.  You shouldn't normally need to use this but it is a way to access a specific photo if you know the UUID.  If more than more uuid is provided, returns photos that match any of the uuids (e.g. treated as "or")
 - ```persons```: list of one or more persons. Returns only photos containing the person(s).  If more than one person provided, returns photos that match any of the persons (e.g. treated as "or")
 - ```albums```: list of one or more album names.  Returns only photos contained in the album(s). If more than one album name is provided, returns photos contained in any of the albums (.e.g. treated as "or")
+- ```images```: bool; if True, returns photos/images; default is True
+- ```movies```: bool; if True, returns movies/videos; default is False
 
-If more than one of these parameters is provided, they are treated as "and" criteria. E.g.
+If more than one of (keywords, uuid, persons, albums) is provided, they are treated as "and" criteria. E.g.
 
 Finds all photos with (keyword = "wedding" or "birthday") and (persons = "Juan Rodriguez")
 
@@ -447,6 +454,16 @@ photos2 = photosdb.photos(keywords=["Kids"])
 photos3 = [p for p in photos2 if p not in photos1]
 ```
 
+By default, photos() only returns images, not movies.  To also get movies, pass movies=True:
+```python
+photos_and_movies = photosdb.photos(movies=True)
+```
+
+To get only movies:
+```python
+movies = photosdb.photos(images=False, movies=True)
+```
+
 ### PhotoInfo 
 PhotosDB.photos() returns a list of PhotoInfo objects.  Each PhotoInfo object represents a single photo in the Photos library.
 
@@ -506,6 +523,15 @@ Returns True if photo is in a shared album, otherwise False.
 
 **Note**: *Only valid on Photos 5 / MacOS 10.15*; on Photos <= 4, returns None instead of True/False. 
 
+#### `isphoto`
+Returns True if type is photo/still image, otherwise False
+
+#### `ismovie`
+Returns True if type is movie/video, otherwise False
+
+#### `uti`
+Returns Uniform Type Identifier (UTI) for the image, for example: 'public.jpeg' or 'com.apple.quicktime-movie'
+
 #### `json()`
 Returns a JSON representation of all photo info 
 

osxphotos/__init__.py

@@ -3,6 +3,7 @@ import logging
 from ._version import __version__
 from .photoinfo import PhotoInfo
 from .photosdb import PhotosDB
+from .utils import _set_debug, _debug, _get_logger
 
 # TODO: find edited photos: see https://github.com/orangeturtle739/photos-export/blob/master/extract_photos.py
 # TODO: Add test for imageTimeZoneOffsetSeconds = None
@@ -13,31 +14,3 @@ from .photosdb import PhotosDB
 # TODO: Add special albums and magic albums
 # TODO: cleanup os.path and pathlib code (import pathlib and also from pathlib import Path)
 
-
-# set _DEBUG = True to enable debug output
-_DEBUG = False
-
-logging.basicConfig(
-    level=logging.DEBUG,
-    format="%(asctime)s - %(levelname)s - %(filename)s - %(lineno)d - %(message)s",
-)
-
-if not _DEBUG:
-    logging.disable(logging.DEBUG)
-
-
-def _get_logger():
-    """Used only for testing
-    
-    Returns:
-        logging.Logger object -- logging.Logger object for osxphotos
-    """
-    return logging.Logger(__name__)
-
-
-def _debug(debug):
-    """ Enable or disable debug logging """
-    if debug:
-        logging.disable(logging.NOTSET)
-    else:
-        logging.disable(logging.DEBUG)

osxphotos/__main__.py

@@ -16,12 +16,12 @@ from ._version import __version__
 from .utils import create_path_by_date
 
 # TODO: add "--any" to search any field (e.g. keyword, description, title contains "wedding") (add case insensitive option)
-
+# TODO: add search for filename
 
 class CLI_Obj:
     def __init__(self, db=None, json=False, debug=False):
         if debug:
-            osxphotos._debug(True)
+            osxphotos._set_debug(True)
         self.db = db
         self.json = json
 
@@ -106,6 +106,14 @@ def info(cli_obj):
     movies = pdb.photos(images=False, movies=True)
     info["movie_count"] = len(movies)
 
+    if pdb.db_version >= _PHOTOS_5_VERSION:
+        shared_photos = [p for p in photos if p.shared]
+        info["shared_photo_count"] = len(shared_photos)
+
+        shared_movies = [p for p in movies if p.shared]
+        info["shared_movie_count"] = len(shared_movies)
+    
+
     keywords = pdb.keywords_as_dict
     info["keywords_count"] = len(keywords)
     info["keywords"] = keywords

osxphotos/_constants.py

@@ -2,6 +2,7 @@
 Constants used by osxphotos 
 """
 
+
 # which Photos library database versions have been tested
 # Photos 2.0 (10.12.6) == 2622
 # Photos 3.0 (10.13.6) == 3301
@@ -28,3 +29,4 @@ _PHOTOS_5_SHARED_PHOTO_PATH = "resources/cloudsharing/data"
 # What type of file? Based on ZGENERICASSET.ZKIND in Photos 5 database
 _PHOTO_TYPE = 0
 _MOVIE_TYPE = 1
+

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.18.03"
+__version__ = "0.19.00"

osxphotos/photoinfo.py

@@ -83,20 +83,6 @@ class PhotoInfo:
             return photopath
             # TODO: Is there a way to use applescript or PhotoKit to force the download in this
 
-        if self._info["masterFingerprint"]:
-            # if masterFingerprint is not null, path appears to be valid
-            if self._info["directory"].startswith("/"):
-                photopath = os.path.join(
-                    self._info["directory"], self._info["filename"]
-                )
-            else:
-                photopath = os.path.join(
-                    self._db._masters_path,
-                    self._info["directory"],
-                    self._info["filename"],
-                )
-            return photopath
-
         if self._info["shared"]:
             # shared photo
             photopath = os.path.join(
@@ -107,13 +93,23 @@ class PhotoInfo:
             )
             return photopath
 
-        # if all else fails, photopath = None
+        # if self._info["masterFingerprint"]:
-        photopath = None
+        # if masterFingerprint is not null, path appears to be valid
-        logging.debug(
+        if self._info["directory"].startswith("/"):
-            f"WARNING: photopath None, masterFingerprint null, not shared {pformat(self._info)}"
+            photopath = os.path.join(self._info["directory"], self._info["filename"])
-        )
+        else:
+            photopath = os.path.join(
+                self._db._masters_path, self._info["directory"], self._info["filename"]
+            )
         return photopath
 
+        # if all else fails, photopath = None
+        # photopath = None
+        # logging.debug(
+        #     f"WARNING: photopath None, masterFingerprint null, not shared {pformat(self._info)}"
+        # )
+        # return photopath
+
     @property
     def path_edited(self):
         """ absolute path on disk of the edited picture """
@@ -176,12 +172,20 @@ class PhotoInfo:
             if self._info["hasAdjustments"]:
                 library = self._db._library_path
                 directory = self._uuid[0]  # first char of uuid
+                filename = None
+                if self._info["type"] == _PHOTO_TYPE:
+                    # it's a photo
+                    filename = f"{self._uuid}_1_201_a.jpeg"
+                elif self._info["type"] == _MOVIE_TYPE:
+                    # it's a movie
+                    filename = f"{self._uuid}_2_0_a.mov"
+                else:
+                    # don't know what it is!
+                    logging.debug(f"WARNING: unknown type {self._info['type']}")
+                    return None
+
                 photopath = os.path.join(
-                    library,
+                    library, "resources", "renders", directory, filename
-                    "resources",
-                    "renders",
-                    directory,
-                    f"{self._uuid}_1_201_a.jpeg",
                 )
 
                 if not os.path.isfile(photopath):

osxphotos/photosdb.py

@@ -16,16 +16,16 @@ from pprint import pformat
 from shutil import copyfile
 
 from ._constants import (
+    _MOVIE_TYPE,
+    _PHOTO_TYPE,
     _PHOTOS_5_VERSION,
     _TESTED_DB_VERSIONS,
     _TESTED_OS_VERSIONS,
     _UNKNOWN_PERSON,
-    _PHOTO_TYPE,
-    _MOVIE_TYPE,
 )
 from ._version import __version__
 from .photoinfo import PhotoInfo
-from .utils import _check_file_exists, _get_os_version, get_last_library_path
+from .utils import _check_file_exists, _get_os_version, get_last_library_path, _debug
 
 # TODO: Add test for imageTimeZoneOffsetSeconds = None
 # TODO: Fix command line so multiple --keyword, etc. are AND (instead of OR as they are in .photos())
@@ -83,7 +83,8 @@ class PhotosDB:
         # list of temporary files created so we can clean them up later
         self._tmp_files = []
 
-        logging.debug(f"dbfile = {dbfile}")
+        if _debug():
+            logging.debug(f"dbfile = {dbfile}")
 
         # get the path to photos library database
         if args:
@@ -117,7 +118,8 @@ class PhotosDB:
         if not _check_file_exists(dbfile):
             raise FileNotFoundError(f"dbfile {dbfile} does not exist", dbfile)
 
-        logging.debug(f"dbfile = {dbfile}")
+        if _debug():
+            logging.debug(f"dbfile = {dbfile}")
 
         self._dbfile = self._dbfile_actual = os.path.abspath(dbfile)
 
@@ -126,7 +128,8 @@ class PhotosDB:
 
         # If Photos >= 5, actual data isn't in photos.db but in Photos.sqlite
         if int(self._db_version) >= int(_PHOTOS_5_VERSION):
-            logging.debug(f"version is {self._db_version}")
+            if _debug():
+                logging.debug(f"version is {self._db_version}")
             dbpath = pathlib.Path(self._dbfile).parent
             dbfile = dbpath / "Photos.sqlite"
             if not _check_file_exists(dbfile):
@@ -134,9 +137,10 @@ class PhotosDB:
             else:
                 self._tmp_db = self._copy_db_file(dbfile)
                 self._dbfile_actual = dbfile
-            logging.debug(
+            if _debug():
-                f"_dbfile = {self._dbfile}, _dbfile_actual = {self._dbfile_actual}"
+                logging.debug(
-            )
+                    f"_dbfile = {self._dbfile}, _dbfile_actual = {self._dbfile_actual}"
+                )
 
         library_path = os.path.dirname(os.path.abspath(dbfile))
         (library_path, _) = os.path.split(library_path)  # drop /database from path
@@ -148,7 +152,8 @@ class PhotosDB:
             masters_path = os.path.join(library_path, "originals")
             self._masters_path = masters_path
 
-        logging.debug(f"library = {library_path}, masters = {masters_path}")
+        if _debug():
+            logging.debug(f"library = {library_path}, masters = {masters_path}")
 
         if int(self._db_version) < int(_PHOTOS_5_VERSION):
             self._process_database4()
@@ -162,12 +167,14 @@ class PhotosDB:
         # logging.debug(f"tmp files = {self._tmp_files}")
         for f in self._tmp_files:
             if os.path.exists(f):
-                logging.debug(f"cleaning up {f}")
+                if _debug():
+                    logging.debug(f"cleaning up {f}")
                 try:
                     os.remove(f)
                     self._tmp_files.remove(f)
                 except Exception as e:
-                    logging.debug("exception %e removing %s" % (e, f))
+                    if _debug():
+                        logging.debug("exception %e removing %s" % (e, f))
             else:
                 self._tmp_files.remove(f)
 
@@ -334,7 +341,8 @@ class PhotosDB:
             raise Exception
 
         self._tmp_files.extend(tmp_files)
-        logging.debug(self._tmp_files)
+        if _debug():
+            logging.debug(self._tmp_files)
 
         return tmp
 
@@ -437,10 +445,11 @@ class PhotosDB:
                 "cloudownerhashedpersonid": None,  # Photos 5
             }
 
-        logging.debug(f"Finished walking through albums")
+        if _debug():
-        logging.debug(pformat(self._dbalbums_album))
+            logging.debug(f"Finished walking through albums")
-        logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbums_album))
-        logging.debug(pformat(self._dbalbum_details))
+            logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbum_details))
 
         # Get info on keywords
         c.execute(
@@ -504,7 +513,8 @@ class PhotosDB:
 
         for row in c:
             uuid = row[0]
-            logging.debug(f"uuid = '{uuid}, master = '{row[2]}")
+            if _debug():
+                logging.debug(f"uuid = '{uuid}, master = '{row[2]}")
             self._dbphotos[uuid] = {}
             self._dbphotos[uuid]["_uuid"] = uuid  # stored here for easier debugging
             self._dbphotos[uuid]["modelID"] = row[1]
@@ -549,7 +559,8 @@ class PhotosDB:
                 self._dbphotos[uuid]["type"] = _MOVIE_TYPE
             else:
                 # unknown
-                logging.debug(f"WARNING: {uuid} found unknown type {row[21]}")
+                if _debug():
+                    logging.debug(f"WARNING: {uuid} found unknown type {row[21]}")
                 self._dbphotos[uuid]["type"] = None
 
             self._dbphotos[uuid]["UTI"] = row[22]
@@ -591,10 +602,11 @@ class PhotosDB:
                         and row[6] == 2
                     ):
                         if "edit_resource_id" in self._dbphotos[uuid]:
-                            logging.debug(
+                            if _debug():
-                                f"WARNING: found more than one edit_resource_id for "
+                                logging.debug(
-                                f"UUID {row[0]},adjustmentUUID {row[1]}, modelID {row[2]}"
+                                    f"WARNING: found more than one edit_resource_id for "
-                            )
+                                    f"UUID {row[0]},adjustmentUUID {row[1]}, modelID {row[2]}"
+                                )
                         # TODO: I think there should never be more than one edit but
                         # I've seen this once in my library
                         # should we return all edits or just most recent one?
@@ -656,32 +668,34 @@ class PhotosDB:
         # remove temporary files
         self._cleanup_tmp_files()
 
-        logging.debug("Faces:")
+        if _debug():
-        logging.debug(pformat(self._dbfaces_uuid))
+            logging.debug("Faces:")
+            logging.debug(pformat(self._dbfaces_uuid))
 
-        logging.debug("Keywords by uuid:")
+            logging.debug("Keywords by uuid:")
-        logging.debug(pformat(self._dbkeywords_uuid))
+            logging.debug(pformat(self._dbkeywords_uuid))
 
-        logging.debug("Keywords by keyword:")
+            logging.debug("Keywords by keyword:")
-        logging.debug(pformat(self._dbkeywords_keyword))
+            logging.debug(pformat(self._dbkeywords_keyword))
 
-        logging.debug("Albums by uuid:")
+            logging.debug("Albums by uuid:")
-        logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbums_uuid))
 
-        logging.debug("Albums by album:")
+            logging.debug("Albums by album:")
-        logging.debug(pformat(self._dbalbums_album))
+            logging.debug(pformat(self._dbalbums_album))
 
-        logging.debug("Volumes:")
+            logging.debug("Volumes:")
-        logging.debug(pformat(self._dbvolumes))
+            logging.debug(pformat(self._dbvolumes))
 
-        logging.debug("Photos:")
+            logging.debug("Photos:")
-        logging.debug(pformat(self._dbphotos))
+            logging.debug(pformat(self._dbphotos))
 
     def _process_database5(self):
         """ process the Photos database to extract info """
         """ works on Photos version >= 5.0 """
 
-        logging.debug(f"_process_database5")
+        if _debug():
+            logging.debug(f"_process_database5")
 
         # Epoch is Jan 1, 2001
         td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
@@ -689,7 +703,8 @@ class PhotosDB:
         (conn, c) = self._open_sql_file(self._tmp_db)
 
         # Look for all combinations of persons and pictures
-        logging.debug(f"Getting information about persons")
+        if _debug():
+            logging.debug(f"Getting information about persons")
 
         c.execute(
             "SELECT ZPERSON.ZFULLNAME, ZGENERICASSET.ZUUID "
@@ -707,9 +722,11 @@ class PhotosDB:
                 self._dbfaces_person[person_name] = []
             self._dbfaces_uuid[person[1]].append(person_name)
             self._dbfaces_person[person_name].append(person[1])
-        logging.debug(f"Finished walking through persons")
+
-        logging.debug(pformat(self._dbfaces_person))
+        if _debug():
-        logging.debug(self._dbfaces_uuid)
+            logging.debug(f"Finished walking through persons")
+            logging.debug(pformat(self._dbfaces_person))
+            logging.debug(self._dbfaces_uuid)
 
         c.execute(
             "SELECT ZGENERICALBUM.ZUUID, ZGENERICASSET.ZUUID "
@@ -749,10 +766,11 @@ class PhotosDB:
                 "cloudidentifier": None,  # Photos4
             }
 
-        logging.debug(f"Finished walking through albums")
+        if _debug():
-        logging.debug(pformat(self._dbalbums_album))
+            logging.debug(f"Finished walking through albums")
-        logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbums_album))
-        logging.debug(pformat(self._dbalbum_details))
+            logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbum_details))
 
         # get details on keywords
         c.execute(
@@ -770,16 +788,20 @@ class PhotosDB:
                 self._dbkeywords_keyword[keyword[0]] = []
             self._dbkeywords_uuid[keyword[1]].append(keyword[0])
             self._dbkeywords_keyword[keyword[0]].append(keyword[1])
-        logging.debug(f"Finished walking through keywords")
+
-        logging.debug(pformat(self._dbkeywords_keyword))
+        if _debug():
-        logging.debug(pformat(self._dbkeywords_uuid))
+            logging.debug(f"Finished walking through keywords")
+            logging.debug(pformat(self._dbkeywords_keyword))
+            logging.debug(pformat(self._dbkeywords_uuid))
 
         # get details on disk volumes
         c.execute("SELECT ZUUID, ZNAME from ZFILESYSTEMVOLUME")
         for vol in c:
             self._dbvolumes[vol[0]] = vol[1]
-        logging.debug(f"Finished walking through volumes")
+
-        logging.debug(self._dbvolumes)
+        if _debug():
+            logging.debug(f"Finished walking through volumes")
+            logging.debug(self._dbvolumes)
 
         # get details about photos
         logging.debug(f"Getting information about photos")
@@ -800,7 +822,7 @@ class PhotosDB:
             "ZGENERICASSET.ZLATITUDE, "
             "ZGENERICASSET.ZLONGITUDE, "
             "ZGENERICASSET.ZHASADJUSTMENTS, "
-            "ZGENERICASSET.ZCLOUDOWNERHASHEDPERSONID, "
+            "ZGENERICASSET.ZCLOUDBATCHPUBLISHDATE, "
             "ZGENERICASSET.ZKIND, "
             "ZGENERICASSET.ZUNIFORMTYPEIDENTIFIER "
             "FROM ZGENERICASSET "
@@ -825,7 +847,7 @@ class PhotosDB:
         # 13   "ZGENERICASSET.ZLATITUDE, "
         # 14   "ZGENERICASSET.ZLONGITUDE, "
         # 15   "ZGENERICASSET.ZHASADJUSTMENTS "
-        # 16   "ZCLOUDOWNERHASHEDPERSONID "   -- If not null, indicates a shared photo
+        # 16   "ZCLOUDBATCHPUBLISHDATE "   -- If not null, indicates a shared photo
         # 17    "ZKIND," -- 0 = photo, 1 = movie
         # 18    " ZUNIFORMTYPEIDENTIFIER " -- UTI
 
@@ -865,7 +887,7 @@ class PhotosDB:
 
             self._dbphotos[uuid]["hasAdjustments"] = row[15]
 
-            self._dbphotos[uuid]["cloudOwnerHashedPersonID"] = row[16]
+            self._dbphotos[uuid]["cloudbatchpublishdate"] = row[16]
             self._dbphotos[uuid]["shared"] = True if row[16] is not None else False
 
             # these will get filled in later
@@ -883,7 +905,8 @@ class PhotosDB:
             elif row[17] == 1:
                 self._dbphotos[uuid]["type"] = _MOVIE_TYPE
             else:
-                logging.debug(f"WARNING: {uuid} found unknown type {row[17]}")
+                if _debug():
+                    logging.debug(f"WARNING: {uuid} found unknown type {row[17]}")
                 self._dbphotos[uuid]["type"] = None
 
             self._dbphotos[uuid]["UTI"] = row[18]
@@ -902,9 +925,10 @@ class PhotosDB:
             if uuid in self._dbphotos:
                 self._dbphotos[uuid]["extendedDescription"] = row[1]
             else:
-                logging.debug(
+                if _debug():
-                    f"WARNING: found description {row[1]} but no photo for {uuid}"
+                    logging.debug(
-                )
+                        f"WARNING: found description {row[1]} but no photo for {uuid}"
+                    )
 
         # get information about adjusted/edited photos
         c.execute(
@@ -921,32 +945,18 @@ class PhotosDB:
             if uuid in self._dbphotos:
                 self._dbphotos[uuid]["adjustmentFormatID"] = row[2]
             else:
-                logging.debug(
+                if _debug():
-                    f"WARNING: found adjustmentformatidentifier {row[2]} but no photo for uuid {row[0]}"
+                    logging.debug(
-                )
+                        f"WARNING: found adjustmentformatidentifier {row[2]} but no photo for uuid {row[0]}"
+                    )
 
-        # get information on local/remote availability
+        # Find missing photos
-        c.execute(
+        # TODO: this code is very kludgy and I had to make lots of assumptions
-            "SELECT ZGENERICASSET.ZUUID, "
+        # it's probably wrong and needs to be re-worked once I figure out how to reliably
-            "ZINTERNALRESOURCE.ZLOCALAVAILABILITY, "
+        # determine if a photo is missing in Photos 5
-            "ZINTERNALRESOURCE.ZREMOTEAVAILABILITY "
-            "FROM ZGENERICASSET "
-            "JOIN ZADDITIONALASSETATTRIBUTES ON ZADDITIONALASSETATTRIBUTES.ZASSET = ZGENERICASSET.Z_PK "
-            "JOIN ZINTERNALRESOURCE ON ZINTERNALRESOURCE.ZFINGERPRINT = ZADDITIONALASSETATTRIBUTES.ZMASTERFINGERPRINT "
-        )
-
-        for row in c:
-            uuid = row[0]
-            if uuid in self._dbphotos:
-                self._dbphotos[uuid]["localAvailability"] = row[1]
-                self._dbphotos[uuid]["remoteAvailability"] = row[2]
-                if row[1] != 1:
-                    self._dbphotos[uuid]["isMissing"] = 1
-                else:
-                    self._dbphotos[uuid]["isMissing"] = 0
 
         # Get info on remote/local availability for photos in shared albums
-        # Shared photos have a null fingerprint
+        # Shared photos have a null fingerprint (and some other photos do too)
         c.execute(
             """ SELECT 
                 ZGENERICASSET.ZUUID, 
@@ -955,7 +965,37 @@ class PhotosDB:
                 FROM ZGENERICASSET
                 JOIN ZADDITIONALASSETATTRIBUTES ON ZADDITIONALASSETATTRIBUTES.ZASSET = ZGENERICASSET.Z_PK 
                 JOIN ZINTERNALRESOURCE ON ZINTERNALRESOURCE.ZASSET = ZADDITIONALASSETATTRIBUTES.ZASSET 
-                WHERE ZINTERNALRESOURCE.ZFINGERPRINT IS NULL AND ZINTERNALRESOURCE.ZDATASTORESUBTYPE = 3 """
+                WHERE  ZDATASTORESUBTYPE = 0 OR ZDATASTORESUBTYPE = 3 """
+            # WHERE ZINTERNALRESOURCE.ZFINGERPRINT IS NULL AND ZINTERNALRESOURCE.ZDATASTORESUBTYPE = 3 """
+        )
+
+        for row in c:
+            uuid = row[0]
+            if uuid in self._dbphotos:
+                #  and self._dbphotos[uuid]["isMissing"] is None:
+                self._dbphotos[uuid]["localAvailability"] = row[1]
+                self._dbphotos[uuid]["remoteAvailability"] = row[2]
+
+                # old = self._dbphotos[uuid]["isMissing"]
+
+                if row[1] != 1:
+                    self._dbphotos[uuid]["isMissing"] = 1
+                else:
+                    self._dbphotos[uuid]["isMissing"] = 0
+
+                # if old is not None and old != self._dbphotos[uuid]["isMissing"]:
+                #     logging.warning(
+                #         f"{uuid} isMissing changed: {old} {self._dbphotos[uuid]['isMissing']}"
+                #     )
+
+        # get information on local/remote availability
+        c.execute(
+            """ SELECT ZGENERICASSET.ZUUID,
+                ZINTERNALRESOURCE.ZLOCALAVAILABILITY,
+                ZINTERNALRESOURCE.ZREMOTEAVAILABILITY
+                FROM ZGENERICASSET
+                JOIN ZADDITIONALASSETATTRIBUTES ON ZADDITIONALASSETATTRIBUTES.ZASSET = ZGENERICASSET.Z_PK
+                JOIN ZINTERNALRESOURCE ON ZINTERNALRESOURCE.ZFINGERPRINT = ZADDITIONALASSETATTRIBUTES.ZMASTERFINGERPRINT """
         )
 
         for row in c:
@@ -963,12 +1003,21 @@ class PhotosDB:
             if uuid in self._dbphotos:
                 self._dbphotos[uuid]["localAvailability"] = row[1]
                 self._dbphotos[uuid]["remoteAvailability"] = row[2]
+
+                # old = self._dbphotos[uuid]["isMissing"]
+
                 if row[1] != 1:
                     self._dbphotos[uuid]["isMissing"] = 1
                 else:
                     self._dbphotos[uuid]["isMissing"] = 0
 
-        logging.debug(pformat(self._dbphotos))
+                # if old is not None and old != self._dbphotos[uuid]["isMissing"]:
+                #     logging.warning(
+                #         f"{uuid} isMissing changed: {old} {self._dbphotos[uuid]['isMissing']}"
+                #     )
+
+        if _debug():
+            logging.debug(pformat(self._dbphotos))
 
         # add faces and keywords to photo data
         for uuid in self._dbphotos:
@@ -998,26 +1047,30 @@ class PhotosDB:
         conn.close()
         self._cleanup_tmp_files()
 
-        logging.debug("Faces:")
+        if _debug():
-        logging.debug(pformat(self._dbfaces_uuid))
+            logging.debug("Faces:")
+            logging.debug(pformat(self._dbfaces_uuid))
 
-        logging.debug("Keywords by uuid:")
+            logging.debug("Keywords by uuid:")
-        logging.debug(pformat(self._dbkeywords_uuid))
+            logging.debug(pformat(self._dbkeywords_uuid))
 
-        logging.debug("Keywords by keyword:")
+            logging.debug("Keywords by keyword:")
-        logging.debug(pformat(self._dbkeywords_keyword))
+            logging.debug(pformat(self._dbkeywords_keyword))
 
-        logging.debug("Albums by uuid:")
+            logging.debug("Albums by uuid:")
-        logging.debug(pformat(self._dbalbums_uuid))
+            logging.debug(pformat(self._dbalbums_uuid))
 
-        logging.debug("Albums by album:")
+            logging.debug("Albums by album:")
-        logging.debug(pformat(self._dbalbums_album))
+            logging.debug(pformat(self._dbalbums_album))
 
-        logging.debug("Volumes:")
+            logging.debug("Album details:")
-        logging.debug(pformat(self._dbvolumes))
+            logging.debug(pformat(self._dbalbum_details))
 
-        logging.debug("Photos:")
+            logging.debug("Volumes:")
-        logging.debug(pformat(self._dbphotos))
+            logging.debug(pformat(self._dbvolumes))
+
+            logging.debug("Photos:")
+            logging.debug(pformat(self._dbphotos))
 
     # TODO: fix default values to None instead of []
     def photos(

osxphotos/utils.py

@@ -11,6 +11,41 @@ import CoreFoundation
 import objc
 from Foundation import *
 
+_DEBUG = False
+
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(asctime)s - %(levelname)s - %(filename)s - %(lineno)d - %(message)s",
+)
+
+if not _DEBUG:
+    logging.disable(logging.DEBUG)
+
+
+def _get_logger():
+    """Used only for testing
+    
+    Returns:
+        logging.Logger object -- logging.Logger object for osxphotos
+    """
+    return logging.Logger(__name__)
+
+
+def _set_debug(debug):
+    """ Enable or disable debug logging """
+    global _DEBUG
+    _DEBUG = debug
+    if debug:
+        logging.disable(logging.NOTSET)
+    else:
+        logging.disable(logging.DEBUG)
+
+
+def _debug():
+    """ returns True if debugging turned on (via _set_debug), otherwise, false """
+    return _DEBUG
+
 
 def _get_os_version():
     # returns tuple containing OS version

tests/Test-10.12.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist

@@ -3,8 +3,8 @@
 <plist version="1.0">
 <dict>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
-	<date>2019-12-26T14:55:03Z</date>
+	<date>2019-12-28T22:35:14Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2019-12-26T14:55:03Z</date>
+	<date>2019-12-29T08:28:13Z</date>
 </dict>
 </plist>

tests/Test-10.12.6.photoslibrary/resources/moments/analysismetadata.plist

@@ -11,6 +11,6 @@
 	<key>PLLastRevGeoForcedProviderOutOfDateCheckVersionKey</key>
 	<integer>1</integer>
 	<key>PLLastRevGeoVerFileFetchDateKey</key>
-	<date>2019-12-20T15:56:12Z</date>
+	<date>2019-12-28T22:33:47Z</date>
 </dict>
 </plist>

tests/Test-10.15.1.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
 	<key>pid</key>
-	<integer>1986</integer>
+	<integer>16385</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.1.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2019-12-27T04:06:36Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2019-12-27T04:06:36Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-28T23:29:58Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
-	<date>2019-12-27T04:06:44Z</date>
+	<date>2019-12-29T06:18:45Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
-	<date>2019-12-27T04:06:36Z</date>
+	<date>2019-12-28T23:29:57Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-29T06:18:40Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2019-12-27T04:06:37Z</date>
+	<date>2019-12-28T23:29:58Z</date>
 </dict>
 </plist>

tests/Test-10.15.1.photoslibrary/private/com.apple.photoanalysisd/caches/vision/PhotoAnalysisServicePreferences.plist

@@ -3,8 +3,8 @@
 <plist version="1.0">
 <dict>
 	<key>FaceIDModelLastGenerationKey</key>
-	<date>2019-12-27T04:06:38Z</date>
+	<date>2019-12-28T23:29:59Z</date>
 	<key>LastContactClassificationKey</key>
-	<date>2019-12-27T04:06:40Z</date>
+	<date>2019-12-28T23:30:01Z</date>
 </dict>
 </plist>

tests/Test-Movie-5_0.photoslibrary/database/DataModelVersion.plist

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>LibrarySchemaVersion</key>
+	<integer>5001</integer>
+	<key>MetaSchemaVersion</key>
+	<integer>3</integer>
+</dict>
+</plist>

tests/Test-Movie-5_0.photoslibrary/database/Photos.sqlite.lock

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>hostname</key>
+	<string>Rhets-MacBook-Pro.local</string>
+	<key>hostuuid</key>
+	<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
+	<key>pid</key>
+	<integer>433</integer>
+	<key>processname</key>
+	<string>photolibraryd</string>
+	<key>uid</key>
+	<integer>501</integer>
+</dict>
+</plist>

tests/Test-Movie-5_0.photoslibrary/database/search/searchProgress.plist

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>insertAlbum</key>
+	<array/>
+	<key>insertAsset</key>
+	<array/>
+	<key>insertHighlight</key>
+	<array/>
+	<key>insertMemory</key>
+	<array/>
+	<key>insertMoment</key>
+	<array/>
+	<key>removeAlbum</key>
+	<array/>
+	<key>removeAsset</key>
+	<array/>
+	<key>removeHighlight</key>
+	<array/>
+	<key>removeMemory</key>
+	<array/>
+	<key>removeMoment</key>
+	<array/>
+</dict>
+</plist>

tests/Test-Movie-5_0.photoslibrary/database/search/searchSystemInfo.plist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>embeddingVersion</key>
+	<string>1</string>
+	<key>localeIdentifier</key>
+	<string>en_US</string>
+	<key>sceneTaxonomySHA</key>
+	<string>87914a047c69fbe8013fad2c70fa70c6c03b08b56190fe4054c880e6b9f57cc3</string>
+	<key>searchIndexVersion</key>
+	<string>10</string>
+</dict>
+</plist>

tests/Test-Movie-5_0.photoslibrary/private/com.apple.Photos/appPrivateData.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CollapsedSidebarSectionIdentifiers</key>
+	<array/>
+</dict>
+</plist>

tests/Test-Movie-5_0.photoslibrary/private/com.apple.photolibraryd/appPrivateData.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>PLLibraryServicesManager.LocaleIdentifier</key>
+	<string>en_US</string>
+</dict>
+</plist>