Updated setup.py and README with install instructions

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [0.28.2](https://github.com/RhetTbull/osxphotos/compare/v0.28.1...0.28.2)
+
+> 18 April 2020
+
+- Added folder support for Photos <= 4, closes #93 [`#93`](https://github.com/RhetTbull/osxphotos/issues/93)
+- cleaned up SQL statements in _process_database4 [`6f28171`](https://github.com/RhetTbull/osxphotos/commit/6f281711e2001a63ffad076d7b9835272d5d09da)
+- Updated CHANGELOG.md [`1fa9583`](https://github.com/RhetTbull/osxphotos/commit/1fa9583ea689d54d2613a064f1ade25bcdfbf043)
+- Fixed suffix check on export to be case insensitive [`4b30b3b`](https://github.com/RhetTbull/osxphotos/commit/4b30b3b4260e2c7409e18825e5b626efe646db16)
+- test library update [`3bac106`](https://github.com/RhetTbull/osxphotos/commit/3bac106eb7a180e9e39643a89087d92bf2a437d0)
+
 #### [v0.28.1](https://github.com/RhetTbull/osxphotos/compare/v0.27.4...v0.28.1)
 
 > 18 April 2020
@@ -273,7 +283,11 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 - Moved PhotosDB attributes to properties instead of methods [`d95acdf`](https://github.com/RhetTbull/osxphotos/commit/d95acdf9f8764a1720bcba71a6dad29bf668eaf9)
 - changed interface for export, prepped for exiftool_json_sidecar [`1fe8859`](https://github.com/RhetTbull/osxphotos/commit/1fe885962e8a9a420e776bdd3dc640ca143224b2)
 
-#### [v0.15.1](https://github.com/RhetTbull/osxphotos/compare/v0.14.21...v0.15.1)
+#### [v0.15.1](https://github.com/RhetTbull/osxphotos/compare/v0.15.0...v0.15.1)
+
+> 19 April 2020
+
+#### [v0.15.0](https://github.com/RhetTbull/osxphotos/compare/v0.14.21...v0.15.0)
 
 > 14 December 2019
 

README.md

@@ -33,7 +33,7 @@ OSXPhotos provides the ability to interact with and query Apple's Photos.app lib
 
 ## Supported operating systems
 
-Only works on MacOS (aka Mac OS X). Tested on MacOS 10.12.6 / Photos 2.0, 10.13.6 / Photos 3.0, MacOS 10.14.5, 10.14.6 / Photos 4.0, MacOS 10.15.1 / Photos 5.0. Requires python >= 3.6
+Only works on MacOS (aka Mac OS X). Tested on MacOS 10.12.6 / Photos 2.0, 10.13.6 / Photos 3.0, MacOS 10.14.5, 10.14.6 / Photos 4.0, MacOS 10.15.1 / Photos 5.0. Requires python >= 3.6 though if you use `pip` to install, you must use python >= 3.8.  See notes [below](#Installation-instructions)
 
 This package will read Photos databases for any supported version on any supported OS version.  E.g. you can read a database created with Photos 4.0 on MacOS 10.14 on a machine running MacOS 10.12
 
@@ -44,6 +44,14 @@ osxmetadata uses setuptools, thus simply run:
 
 	python3 setup.py install
 
+If you're using python 3.6 or 3.7, you'll need to do this first to get around an issue with bpylist2:
+
+	pip install -r requirements.txt
+
+You can also install directly from [pypi](https://pypi.org/) but you must use python >= 3.8 to avoid an error with bpylist2.  The package will work fine with python 3.6 or 3.7 but I know of no way to get `pip` to install the right dependencies. 
+
+    pip install osxphotos
+	
 ## Command Line Usage
 
 This package will install a command line utility called `osxphotos` that allows you to query the Photos database.  Alternatively, you can also run the command line utility like this: `python3 -m osxphotos`
@@ -276,7 +284,7 @@ rendered name, use double braces, e.g. '{{' or '}}', thus using
 You may specify an optional default value to use if the substitution does not
 contain a value (e.g. the value is null) by specifying the default value after
 a ',' in the template string: for example, if template is
-'{created.year}/{place.address,'NO_ADDRESS'}' but there was no address
+'{created.year}/{place.address,NO_ADDRESS}' but there was no address
 associated with the photo, the resulting output would be:
 '2020/NO_ADDRESS/photoname.jpg'. If specified, the default value may not
 contain a brace symbol ('{' or '}').

cli.py

@@ -0,0 +1,19 @@
+""" stand alone command line script for use with pyinstaller
+    
+    To build this into an executable:
+    - install pyinstaller:
+        python3 -m pip install pyinstaller
+    - then use build_cli_exe.sh to run pyinstaller or execute the following command:
+        pyinstaller --onefile --hidden-import="pkg_resources.py2_warn" --name osxphotos --add-data osxphotos/templates/xmp_sidecar.mako:osxphotos/templates cli.py
+
+    Resulting executable will be in "dist/osxphotos"
+
+    Note: This is *not* the cli that "python3 -m pip install osxphotos" or "python setup.py install" would install;
+    it's merely a wrapper around __main__.py to allow pyinstaller to work
+    
+"""
+
+from osxphotos.__main__ import cli
+
+if __name__ == "__main__":
+    cli()

make_cli_exe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh 
+
+# This will build an stand-alone executable called 'osxphotos' in your ./dist directory
+# using pyinstaller
+# If you need to install pyinstaller:
+# python3 -m pip install --upgrade pyinstaller
+
+pyinstaller --onefile --hidden-import="pkg_resources.py2_warn" --name osxphotos --add-data osxphotos/templates/xmp_sidecar.mako:osxphotos/templates cli.py

osxphotos/__main__.py

@@ -18,7 +18,7 @@ from pathvalidate import (
 
 import osxphotos
 
-from ._constants import _EXIF_TOOL_URL, _PHOTOS_5_VERSION, _UNKNOWN_PLACE
+from ._constants import _EXIF_TOOL_URL, _PHOTOS_4_VERSION, _UNKNOWN_PLACE
 from ._version import __version__
 from .exiftool import get_exiftool_path
 from .template import (
@@ -109,7 +109,7 @@ class ExportCommand(click.Command):
             "You may specify an optional default value to use if the substitution does not contain a value "
             + "(e.g. the value is null) "
             + "by specifying the default value after a ',' in the template string: "
-            + "for example, if template is '{created.year}/{place.address,'NO_ADDRESS'}' "
+            + "for example, if template is '{created.year}/{place.address,NO_ADDRESS}' "
             + "but there was no address associated with the photo, the resulting output would be: "
             + "'2020/NO_ADDRESS/photoname.jpg'. "
             + "If specified, the default value may not contain a brace symbol ('{' or '}')."
@@ -428,7 +428,7 @@ def albums(ctx, cli_obj, db, json_, photos_library):
 
     photosdb = osxphotos.PhotosDB(dbfile=db)
     albums = {"albums": photosdb.albums_as_dict}
-    if photosdb.db_version >= _PHOTOS_5_VERSION:
+    if photosdb.db_version > _PHOTOS_4_VERSION:
         albums["shared albums"] = photosdb.albums_shared_as_dict
 
     if json_ or cli_obj.json:
@@ -493,7 +493,7 @@ def info(ctx, cli_obj, db, json_, photos_library):
     not_shared_movies = [p for p in movies if not p.shared]
     info["movie_count"] = len(not_shared_movies)
 
-    if pdb.db_version >= _PHOTOS_5_VERSION:
+    if pdb.db_version > _PHOTOS_4_VERSION:
         shared_photos = [p for p in photos if p.shared]
         info["shared_photo_count"] = len(shared_photos)
 
@@ -508,7 +508,7 @@ def info(ctx, cli_obj, db, json_, photos_library):
     info["albums_count"] = len(albums)
     info["albums"] = albums
 
-    if pdb.db_version >= _PHOTOS_5_VERSION:
+    if pdb.db_version > _PHOTOS_4_VERSION:
         albums_shared = pdb.albums_shared_as_dict
         info["shared_albums_count"] = len(albums_shared)
         info["shared_albums"] = albums_shared

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.28.2"
+__version__ = "0.28.5"

osxphotos/photoinfo.py

@@ -53,7 +53,12 @@ class PhotoInfo:
     @property
     def filename(self):
         """ filename of the picture """
-        return self._info["filename"]
+        if self.has_raw and self.raw_original:
+            # return name of the RAW file
+            # TODO: not yet implemented
+            return self._info["filename"]
+        else:
+            return self._info["filename"]
 
     @property
     def original_filename(self):
@@ -296,7 +301,10 @@ class PhotoInfo:
             glob_str = f"{filestem}*.{raw_ext}"
             raw_file = findfiles(glob_str, filepath)
             if len(raw_file) != 1:
-                logging.warning(
+                # Note: In Photos Version 5.0 (141.19.150), images not copied to Photos Library
+                # that are missing do not always trigger is_missing = True as happens
+                # in earlier version so it's possible for this check to fail, if so, return None
+                logging.debug(
                     f"Error getting path to RAW file: {filepath}/{glob_str}"
                 )
                 photopath = None
@@ -610,7 +618,7 @@ class PhotoInfo:
         """ returns True if associated RAW image and the RAW image is selected in Photos
             via "Use RAW as Original "
             otherwise returns False """
-        return True if self._info["original_resource_choice"] == 1 else False
+        return self._info["raw_is_original"]
 
     def export(
         self,

osxphotos/photosdb.py

@@ -240,7 +240,7 @@ class PhotosDB:
         self._db_version = self._get_db_version()
 
         # If Photos >= 5, actual data isn't in photos.db but in Photos.sqlite
-        if int(self._db_version) >= int(_PHOTOS_5_VERSION):
+        if int(self._db_version) > int(_PHOTOS_4_VERSION):
             dbpath = pathlib.Path(self._dbfile).parent
             dbfile = dbpath / "Photos.sqlite"
             if not _check_file_exists(dbfile):
@@ -259,7 +259,7 @@ class PhotosDB:
         library_path = os.path.dirname(os.path.abspath(dbfile))
         (library_path, _) = os.path.split(library_path)  # drop /database from path
         self._library_path = library_path
-        if int(self._db_version) < int(_PHOTOS_5_VERSION):
+        if int(self._db_version) <= int(_PHOTOS_4_VERSION):
             masters_path = os.path.join(library_path, "Masters")
             self._masters_path = masters_path
         else:
@@ -528,7 +528,6 @@ class PhotosDB:
         """ process the Photos database to extract info
             works on Photos version <= 4.0 """
 
-        # TODO: Update strings to remove + (not needed)
         # Epoch is Jan 1, 2001
         td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
 
@@ -888,6 +887,7 @@ class PhotosDB:
             # TODO: NOT YET USED -- PLACEHOLDER for RAW processing (currently only in _process_database5)
             # original resource choice (e.g. RAW or jpeg)
             self._dbphotos[uuid]["original_resource_choice"] = None
+            self._dbphotos[uuid]["raw_is_original"] = None
 
             # associated RAW image info
             self._dbphotos[uuid]["has_raw"] = True if row[25] == 7 else False
@@ -903,17 +903,17 @@ class PhotosDB:
         # get additional details from RKMaster, needed for RAW processing
         c.execute(
             """ SELECT 
-	            RKMaster.uuid,
+                RKMaster.uuid,
                 RKMaster.volumeId, 
                 RKMaster.imagePath, 
-	            RKMaster.isMissing, 
+                RKMaster.isMissing, 
                 RKMaster.originalFileName, 
                 RKMaster.UTI,
-	            RKMaster.modelID, 
+                RKMaster.modelID, 
                 RKMaster.fileSize, 
                 RKMaster.isTrulyRaw,
-	            RKMaster.alternateMasterUuid
-	            FROM RKMaster
+                RKMaster.alternateMasterUuid
+                FROM RKMaster
             """
         )
 
@@ -1607,7 +1607,12 @@ class PhotosDB:
             info["momentID"] = row[26]
 
             # original resource choice (e.g. RAW or jpeg)
+            # for images part of a RAW/jpeg pair, 
+            # ZADDITIONALASSETATTRIBUTES.ZORIGINALRESOURCECHOICE 
+            # = 0 if jpeg is selected as "original" in Photos (the default)
+            # = 1 if RAW is selected as "original" in Photos
             info["original_resource_choice"] = row[27]
+            info["raw_is_original"] = True if row[27] == 1 else False
 
             # associated RAW image info
             # will be filled in later
@@ -1878,7 +1883,7 @@ class PhotosDB:
             # folder with no parent (e.g. shared iCloud folders)
             return folders
 
-        if self._db_version >= _PHOTOS_5_VERSION and parent == self._folder_root_pk:
+        if self._db_version > _PHOTOS_4_VERSION and parent == self._folder_root_pk:
             # at the top of the folder hierarchy, we're done
             return folders
 
@@ -2156,3 +2161,7 @@ class PhotosDB:
             return self.__dict__ == other.__dict__
 
         return False
+
+    def __len__(self):
+        """ returns number of photos in the database """
+        return len(self._dbphotos)

setup.py

@@ -3,7 +3,7 @@
 #
 # setup.py script for osxphotos
 #
-# Copyright (c) 2019 Rhet Turnbull, rturnbull+git@gmail.com
+# Copyright (c) 2019, 2020 Rhet Turnbull, rturnbull+git@gmail.com
 # All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person
@@ -27,23 +27,43 @@
 # SOFTWARE.
 
 import os
+import platform
+
 from setuptools import find_packages, setup
 
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
-    long_description = f.read()
+# python version as 2-digit float (e.g. 3.6)
+py_ver = float(".".join(platform.python_version_tuple()[:2]))
 
+# holds config info read from disk
 about = {}
+this_directory = os.path.abspath(os.path.dirname(__file__))
+
+# get version info from _version
 with open(
     os.path.join(this_directory, "osxphotos", "_version.py"), mode="r", encoding="utf-8"
 ) as f:
     exec(f.read(), about)
 
+# read README.md into long_description
+with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
+    about["long_description"] = f.read()
+
+# ugly hack to install custom version of bpylist2 needed for Python < 3.8
+# the stock version of bylist2==2.0.3 causes an error related to
+# "pkg_resources.ContextualVersionConflict: (pycodestyle 2.3.1..."
+# PEP 508 no help here as URL-based lookups not allowed in PyPI packages
+# if you know a better way, PRs welcome!
+# once I go to 3.8+ required, this won't be necessary as bpylist2 3.0+ solves this issue
+if py_ver < 3.8:
+    os.system(
+        "python3 -m pip install git+git://github.com/RhetTbull/bpylist2.git#egg=bpylist2"
+    )
+
 setup(
     name="osxphotos",
     version=about["__version__"],
     description="Manipulate (read-only) Apple's Photos app library on Mac OS X",
-    long_description=long_description,
+    long_description=about["long_description"],
     long_description_content_type="text/markdown",
     author="Rhet Turnbull",
     author_email="rturnbull+git@gmail.com",
@@ -58,7 +78,7 @@ setup(
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Operating System :: MacOS :: MacOS X",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.8",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
     install_requires=[

tests/Test-10.15.4.photoslibrary/database/Photos.sqlite.lock

@@ -7,7 +7,7 @@
 	<key>hostuuid</key>
 	<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
 	<key>pid</key>
-	<integer>3212</integer>
+	<integer>434</integer>
 	<key>processname</key>
 	<string>photolibraryd</string>
 	<key>uid</key>

tests/Test-10.15.4.photoslibrary/private/com.apple.photoanalysisd/caches/graph/PhotoAnalysisServicePreferences.plist

@@ -3,24 +3,24 @@
 <plist version="1.0">
 <dict>
 	<key>BackgroundHighlightCollection</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:34Z</date>
 	<key>BackgroundHighlightEnrichment</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:34Z</date>
 	<key>BackgroundJobAssetRevGeocode</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:35Z</date>
 	<key>BackgroundJobSearch</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:35Z</date>
 	<key>BackgroundPeopleSuggestion</key>
-	<date>2020-04-11T20:00:24Z</date>
+	<date>2020-04-19T15:27:34Z</date>
 	<key>BackgroundUserBehaviorProcessor</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:35Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphConsistencyUpdateJobDateKey</key>
 	<date>2020-04-11T20:10:27Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
 	<date>2020-04-11T20:00:24Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:35Z</date>
 	<key>SiriPortraitDonation</key>
-	<date>2020-04-11T20:00:25Z</date>
+	<date>2020-04-19T15:27:35Z</date>
 </dict>
 </plist>

tests/test_catalina_10_15_4.py

@@ -118,6 +118,13 @@ def test_init5():
     with pytest.raises(Exception):
         assert osxphotos.PhotosDB()
 
+def test_db_len():
+    import osxphotos
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    # assert photosdb.db_version in osxphotos._TESTED_DB_VERSIONS
+    assert len(photosdb) == 12 
+
 
 def test_db_version():
     import osxphotos
@@ -368,7 +375,7 @@ def test_count():
 
     photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
     photos = photosdb.photos()
-    assert len(photos) == 8
+    assert len(photos) == 12 
 
 
 def test_keyword_2():
@@ -771,7 +778,7 @@ def test_from_to_date():
     photosdb = osxphotos.PhotosDB(PHOTOS_DB)
 
     photos = photosdb.photos(from_date=dt.datetime(2018, 10, 28))
-    assert len(photos) == 2
+    assert len(photos) ==6 
 
     photos = photosdb.photos(to_date=dt.datetime(2018, 10, 28))
     assert len(photos) == 6

tests/test_cli.py

@@ -46,16 +46,22 @@ CLI_EXPORT_FILENAMES = [
 
 CLI_EXPORT_FILENAMES_CURRENT = [
     "1EB2B765-0765-43BA-A90C-0D0580E6172C.jpeg",
+    "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg",
+    "4D521201-92AC-43E5-8F7C-59BC41C37A96.cr2",
+    "4D521201-92AC-43E5-8F7C-59BC41C37A96.jpeg",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4.jpeg",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.cr2",
+    "A92D9C26-3A50-4197-9388-CB5F7DB9FA91.jpeg",
+    "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068.dng",
+    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
     "DC99FBDD-7A52-4100-A5BB-344131646C30.jpeg",
     "DC99FBDD-7A52-4100-A5BB-344131646C30_edited.jpeg",
     "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51.jpeg",
     "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51_edited.jpeg",
-    "D79B8D77-BFFC-460B-9312-034F2877D35B.jpeg",
     "F12384F6-CD17-4151-ACBA-AE0E3688539E.jpeg",
-    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4.jpeg",
-    "3DD2C897-F19E-4CA6-8C22-B027D5A71907.jpeg",
 ]
 
+
 CLI_EXPORTED_DIRECTORY_TEMPLATE_FILENAMES1 = [
     "2019/April/wedding.jpg",
     "2019/July/Tulips.jpg",

tests/test_mojave_10_14_6.py

@@ -58,6 +58,13 @@ def test_db_version():
     assert photosdb.db_version in osxphotos._constants._TESTED_DB_VERSIONS
     assert photosdb.db_version == "4025"
 
+def test_db_len():
+    import osxphotos
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    # assert photosdb.db_version in osxphotos._TESTED_DB_VERSIONS
+    assert len(photosdb) == 7
+
 
 def test_os_version():
     import osxphotos