Fix for issue #353, #354

Force cleanup of objects in write_jpeg (fix memory leak)

.all-contributorsrc

@@ -0,0 +1,173 @@
+{
+  "projectName": "osxphotos",
+  "projectOwner": "RhetTbull",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 75,
+  "badgeTemplate": "[![All Contributors](https://img.shields.io/badge/all_contributors-<%= contributors.length %>-orange.svg?style=flat)](#contributors)",
+  "commit": true,
+  "commitConvention": "none",
+  "contributors": [
+    {
+      "login": "britiscurious",
+      "name": "britiscurious",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/25646439?v=4",
+      "profile": "https://github.com/britiscurious",
+      "contributions": [
+        "doc",
+        "code"
+      ]
+    },
+    {
+      "login": "mwort",
+      "name": "Michel Wortmann",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/8170417?v=4",
+      "profile": "https://github.com/mwort",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "PabloKohan",
+      "name": "Pablo 'merKur' Kohan",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/8790976?v=4",
+      "profile": "https://github.com/PabloKohan",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "hshore29",
+      "name": "hshore29",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/7023497?v=4",
+      "profile": "https://github.com/hshore29",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "dmd",
+      "name": "Daniel M. Drucker",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/41439?v=4",
+      "profile": "http://3e.org/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "jystervinou",
+      "name": "Jean-Yves Stervinou",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/132356?v=4",
+      "profile": "https://github.com/jystervinou",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "dethi",
+      "name": "Thibault Deutsch",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/1011520?v=4",
+      "profile": "https://dethi.me/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "grundsch",
+      "name": "grundsch",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/3874928?v=4",
+      "profile": "https://github.com/grundsch",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "agprimatic",
+      "name": "Ag Primatic",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/4685054?v=4",
+      "profile": "https://github.com/agprimatic",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "hhoeck",
+      "name": "Horst Höck",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/6313998?v=4",
+      "profile": "https://github.com/hhoeck",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "jstrine",
+      "name": "Jonathan Strine",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/33943447?v=4",
+      "profile": "https://github.com/jstrine",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "finestream",
+      "name": "finestream",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/16638513?v=4",
+      "profile": "https://github.com/finestream",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "synox",
+      "name": "Aravindo Wingeier",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/2250964?v=4",
+      "profile": "https://github.com/synox",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "kradalby",
+      "name": "Kristoffer Dalby",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/98431?v=4",
+      "profile": "https://kradalby.no",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "Rott-Apple",
+      "name": "Rott-Apple",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/67875570?v=4",
+      "profile": "https://github.com/Rott-Apple",
+      "contributions": [
+        "research"
+      ]
+    },
+    {
+      "login": "narensankar0529",
+      "name": "narensankar0529",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/74054766?v=4",
+      "profile": "https://github.com/narensankar0529",
+      "contributions": [
+        "bug",
+        "userTesting"
+      ]
+    },
+    {
+      "login": "martinhrpi",
+      "name": "Martin",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/19407684?v=4",
+      "profile": "https://github.com/martinhrpi",
+      "contributions": [
+        "research",
+        "userTesting"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true
+}

.github/workflows/tests.yml

@@ -1,15 +1,16 @@
-name: Python package
+name: Tests
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
 
     runs-on: macOS-latest
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8]
+        python-version: [3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v1

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Rhet Turnbull
+Copyright (c) 2019-2021 Rhet Turnbull
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -0,0 +1,271 @@
+.. role:: raw-html-m2r(raw)
+   :format: html
+
+
+OSXPhotos
+=========
+
+What is osxphotos?
+------------------
+
+OSXPhotos provides both the ability to interact with and query Apple's Photos.app library on macOS directly from your python code 
+as well as a very flexible command line interface (CLI) app for exporting photos. 
+You can query the Photos library database -- for example, file name, file path, and metadata such as keywords/tags, persons/faces, albums, etc. 
+You can also easily export both the original and edited photos. 
+
+Supported operating systems
+---------------------------
+
+Only works on macOS (aka Mac OS X). Tested on macOS Sierra (10.12.6) until macOS Catalina (10.15.7). 
+Beta support for macOS Big Sur (10.16.01/11.01).
+
+This package will read Photos databases for any supported version on any supported macOS version.  
+E.g. you can read a database created with Photos 5.0 on MacOS 10.15 on a machine running macOS 10.12 and vice versa.
+
+Requires python >= ``3.7``. 
+
+Installation
+------------
+
+If you are new to python and just want to use the command line application, I recommend you to install using pipx. See other advanced options below. 
+
+Installation using pipx
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If you aren't familiar with installing python applications, I recommend you install ``osxphotos`` with `pipx <https://github.com/pipxproject/pipx>`_. If you use ``pipx``\ , you will not need to create a virtual environment as ``pipx`` takes care of this. The easiest way to do this on a Mac is to use `homebrew <https://brew.sh/>`_\ :
+
+
+* Open ``Terminal`` (search for ``Terminal`` in Spotlight or look in ``Applications/Utilities``\ )
+* Install ``homebrew`` according to instructions at `https://brew.sh/ <https://brew.sh/>`_
+* Type the following into Terminal: ``brew install pipx``
+* Then type this: ``pipx install osxphotos``
+* Now you should be able to run ``osxphotos`` by typing: ``osxphotos``
+
+Installation using pip
+^^^^^^^^^^^^^^^^^^^^^^
+
+You can also install directly from `pypi <https://pypi.org/project/osxphotos/>`_\ :
+
+.. code-block::
+
+   pip install osxphotos
+
+
+Installation from git repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+OSXPhotos uses setuptools, thus simply run:
+
+.. code-block::
+
+   git clone https://github.com/RhetTbull/osxphotos.git
+   cd osxphotos
+   python3 setup.py install
+
+
+I recommend you create a `virtual environment <https://docs.python.org/3/tutorial/venv.html>`_ before installing osxphotos.
+
+**WARNING** The git repo for this project is very large (> 1GB) because it contains multiple Photos libraries used for testing 
+on different versions of macOS.  If you just want to use the osxphotos package in your own code, 
+I recommend you install the latest version from `PyPI <https://pypi.org/project/osxphotos/>`_ which does not include all the test 
+libraries. If you just want to use the command line utility, you can download a pre-built executable of the latest 
+`release <https://github.com/RhetTbull/osxphotos/releases>`_ or you can install via ``pip`` which also installs the command line app.  
+If you aren't comfortable with running python on your Mac, start with the pre-built executable or ``pipx`` as described above.
+
+Command Line Usage
+------------------
+
+This package will install a command line utility called ``osxphotos`` that allows you to query the Photos database and export photos.  
+Alternatively, you can also run the command line utility like this: ``python3 -m osxphotos``
+
+.. code-block::
+
+   > osxphotos
+   Usage: osxphotos [OPTIONS] COMMAND [ARGS]...
+
+   Options:
+     --db <Photos database path>  Specify Photos database path. Path to Photos
+                                  library/database can be specified using either
+                                  --db or directly as PHOTOS_LIBRARY positional
+                                  argument. If neither --db or PHOTOS_LIBRARY
+                                  provided, will attempt to find the library to
+                                  use in the following order: 1. last opened
+                                  library, 2. system library, 3.
+                                  ~/Pictures/Photos Library.photoslibrary
+     --json                       Print output in JSON format.
+     -v, --version                Show the version and exit.
+     -h, --help                   Show this message and exit.
+
+   Commands:
+     about     Print information about osxphotos including license.
+     albums    Print out albums found in the Photos library.
+     dump      Print list of all photos & associated info from the Photos...
+     export    Export photos from the Photos database.
+     help      Print help; for help on commands: help <command>.
+     info      Print out descriptive info of the Photos library database.
+     keywords  Print out keywords found in the Photos library.
+     labels    Print out image classification labels found in the Photos...
+     list      Print list of Photos libraries found on the system.
+     persons   Print out persons (faces) found in the Photos library.
+     places    Print out places found in the Photos library.
+     query     Query the Photos database using 1 or more search options; if...
+
+To get help on a specific command, use ``osxphotos help <command_name>``
+
+Command line examples
+^^^^^^^^^^^^^^^^^^^^^
+
+export all photos to ~/Desktop/export group in folders by date created
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``osxphotos export --export-by-date ~/Pictures/Photos\ Library.photoslibrary ~/Desktop/export``
+
+**Note**\ : Photos library/database path can also be specified using ``--db`` option:
+
+``osxphotos export --export-by-date --db ~/Pictures/Photos\ Library.photoslibrary ~/Desktop/export``
+
+find all photos with keyword "Kids" and output results to json file named results.json:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``osxphotos query --keyword Kids --json ~/Pictures/Photos\ Library.photoslibrary >results.json``
+
+export photos to file structure based on 4-digit year and full name of month of photo's creation date:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``osxphotos export ~/Desktop/export --directory "{created.year}/{created.month}"``
+
+(by default, it will attempt to use the system library)
+
+export photos to file structure based on 4-digit year of photo's creation date and add keywords for media type and labels (labels are only awailable on Photos 5 and higher):
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``osxphotos export ~/Desktop/export --directory "{created.year}" --keyword-template "{label}" --keyword-template "{media_type}"`` 
+
+export default library using 'country name/year' as output directory (but use "NoCountry/year" if country not specified), add persons, album names, and year as keywords, write exif metadata to files when exporting, update only changed files, print verbose ouput
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``osxphotos export ~/Desktop/export --directory "{place.name.country,NoCountry}/{created.year}"  --person-keyword --album-keyword --keyword-template "{created.year}" --exiftool --update --verbose``
+
+Example uses of the package
+---------------------------
+
+.. code-block:: python
+
+   """ Simple usage of the package """
+   import osxphotos
+
+   def main():
+       photosdb = osxphotos.PhotosDB()
+       print(photosdb.keywords)
+       print(photosdb.persons)
+       print(photosdb.album_names)
+
+       print(photosdb.keywords_as_dict)
+       print(photosdb.persons_as_dict)
+       print(photosdb.albums_as_dict)
+
+       # find all photos with Keyword = Foo and containing John Smith
+       photos = photosdb.photos(keywords=["Foo"],persons=["John Smith"])
+
+       # find all photos that include Alice Smith but do not contain the keyword Bar
+       photos = [p for p in photosdb.photos(persons=["Alice Smith"]) 
+                   if p not in photosdb.photos(keywords=["Bar"]) ]
+       for p in photos:
+           print(
+               p.uuid,
+               p.filename,
+               p.original_filename,
+               p.date,
+               p.description,
+               p.title,
+               p.keywords,
+               p.albums,
+               p.persons,
+               p.path,
+           )
+
+   if __name__ == "__main__":
+       main()
+
+.. code-block:: python
+
+   """ Export all photos to specified directory using album names as folders
+       If file has been edited, also export the edited version, 
+       otherwise, export the original version 
+       This will result in duplicate photos if photo is in more than album """
+
+   import os.path
+   import pathlib
+   import sys
+
+   import click
+   from pathvalidate import is_valid_filepath, sanitize_filepath
+
+   import osxphotos
+
+
+   @click.command()
+   @click.argument("export_path", type=click.Path(exists=True))
+   @click.option(
+       "--default-album",
+       help="Default folder for photos with no album. Defaults to 'unfiled'",
+       default="unfiled",
+   )
+   @click.option(
+       "--library-path",
+       help="Path to Photos library, default to last used library",
+       default=None,
+   )
+   def export(export_path, default_album, library_path):
+       export_path = os.path.expanduser(export_path)
+       library_path = os.path.expanduser(library_path) if library_path else None
+
+       if library_path is not None:
+           photosdb = osxphotos.PhotosDB(library_path)
+       else:
+           photosdb = osxphotos.PhotosDB()
+
+       photos = photosdb.photos()
+
+       for p in photos:
+           if not p.ismissing:
+               albums = p.albums
+               if not albums:
+                   albums = [default_album]
+               for album in albums:
+                   click.echo(f"exporting {p.filename} in album {album}")
+
+                   # make sure no invalid characters in destination path (could be in album name)
+                   album_name = sanitize_filepath(album, platform="auto")
+
+                   # create destination folder, if necessary, based on album name
+                   dest_dir = os.path.join(export_path, album_name)
+
+                   # verify path is a valid path
+                   if not is_valid_filepath(dest_dir, platform="auto"):
+                       sys.exit(f"Invalid filepath {dest_dir}")
+
+                   # create destination dir if needed
+                   if not os.path.isdir(dest_dir):
+                       os.makedirs(dest_dir)
+
+                   # export the photo
+                   if p.hasadjustments:
+                       # export edited version
+                       exported = p.export(dest_dir, edited=True)
+                       edited_name = pathlib.Path(p.path_edited).name
+                       click.echo(f"Exported {edited_name} to {exported}")
+                   # export unedited version
+                   exported = p.export(dest_dir)
+                   click.echo(f"Exported {p.filename} to {exported}")
+           else:
+               click.echo(f"Skipping missing photo: {p.filename}")
+
+
+   if __name__ == "__main__":
+       export()  # pylint: disable=no-value-for-parameter
+
+Package Interface
+-----------------
+
+Reference full documentation on `GitHub <https://github.com/RhetTbull/osxphotos/blob/master/README.md>`_

cli.py

@@ -3,8 +3,7 @@
     To build this into an executable:
     - install pyinstaller:
         python3 -m pip install pyinstaller
-    - then use make_cli_exe.sh to run pyinstaller or execute the following command:
-        pyinstaller --onefile --hidden-import="pkg_resources.py2_warn" --name osxphotos --add-data osxphotos/templates/xmp_sidecar.mako:osxphotos/templates cli.py
+    - then use make_cli_exe.sh to run pyinstaller
 
     Resulting executable will be in "dist/osxphotos"
 

docs/screencast/terminalizer-demo.yml

@@ -0,0 +1,296 @@
+# how to use this file? see https://github.com/faressoft/terminalizer 
+
+# running commands: 
+# mkdir trip
+# osxphotos export --export-by-date --from-date 2021-01-01 trip 
+# du -h trip
+# find trip | head -20
+
+
+
+# The configurations that used for the recording, feel free to edit them
+config:
+
+  # Specify a command to be executed
+  # like `/bin/bash -l`, `ls`, or any other commands
+  # the default is bash for Linux
+  # or powershell.exe for Windows
+  command: zsh
+  
+  # Specify the current working directory path
+  # the default is the current working directory path
+  cwd: /Users/aravindo/Downloads
+  
+  # Export additional ENV variables
+  env:
+    recording: true
+  
+  # Explicitly set the number of columns
+  # or use `auto` to take the current
+  # number of columns of your shell
+  cols: 91
+  
+  # Explicitly set the number of rows
+  # or use `auto` to take the current
+  # number of rows of your shell
+  rows: 20
+  
+  # Amount of times to repeat GIF
+  # If value is -1, play once
+  # If value is 0, loop indefinitely
+  # If value is a positive number, loop n times
+  repeat: 0
+  
+  # Quality
+  # 1 - 100
+  quality: 100
+  
+  # Delay between frames in ms
+  # If the value is `auto` use the actual recording delays
+  frameDelay: auto
+  
+  # Maximum delay between frames in ms
+  # Ignored if the `frameDelay` isn't set to `auto`
+  # Set to `auto` to prevent limiting the max idle time
+  maxIdleTime: 2000
+  
+  # The surrounding frame box
+  # The `type` can be null, window, floating, or solid`
+  # To hide the title use the value null
+  # Don't forget to add a backgroundColor style with a null as type
+  frameBox:
+    type: floating
+    title: ""
+    style:
+      border: 0px black solid
+      # boxShadow: none
+      # margin: 0px
+  
+  # Add a watermark image to the rendered gif
+  # You need to specify an absolute path for
+  # the image on your machine or a URL, and you can also
+  # add your own CSS styles
+  watermark:
+    imagePath: null
+    style:
+      position: absolute
+      right: 15px
+      bottom: 15px
+      width: 100px
+      opacity: 0.9
+  
+  # Cursor style can be one of
+  # `block`, `underline`, or `bar`
+  cursorStyle: block
+  
+  # Font family
+  # You can use any font that is installed on your machine
+  # in CSS-like syntax
+  fontFamily: "Monaco, Lucida Console, Ubuntu Mono, Monospace"
+  
+  # The size of the font
+  fontSize: 12
+  
+  # The height of lines
+  lineHeight: 1
+  
+  # The spacing between letters
+  letterSpacing: 0
+  
+  # Theme
+  theme:
+    background: "transparent"
+    foreground: "#afafaf"
+    cursor: "#c7c7c7"
+    black: "#232628"
+    red: "#fc4384"
+    green: "#b3e33b"
+    yellow: "#ffa727"
+    blue: "#75dff2"
+    magenta: "#ae89fe"
+    cyan: "#708387"
+    white: "#d5d5d0"
+    brightBlack: "#626566"
+    brightRed: "#ff7fac"
+    brightGreen: "#c8ed71"
+    brightYellow: "#ebdf86"
+    brightBlue: "#75dff2"
+    brightMagenta: "#ae89fe"
+    brightCyan: "#b1c6ca"
+    brightWhite: "#f9f9f4"
+  
+# Records, feel free to edit them
+records:
+  - delay: 100
+    content: "\e[1m\e[7m%\e[27m\e[1m\e[0m                                                                                          \r \r\e]7;file://wingeier-macOS/Users/aravindo/Downloads\a\r\e[0m\e[27m\e[24m\e[J❯ \e[K\e[?2004h"
+  - delay: 100
+    content: m
+  - delay: 100
+    content: "\bmk"
+  - delay: 100
+    content: d
+  - delay: 100
+    content: i
+  - delay: 100
+    content: r
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: t
+  - delay: 100
+    content: r
+  - delay: 100
+    content: i
+  - delay: 100
+    content: p
+  - delay: 100
+    content: "\e[?2004l\r\r\n"
+  - delay: 9
+    content: "\e[1m\e[7m%\e[27m\e[1m\e[0m                                                                                          \r \r\e]7;file://wingeier-macOS/Users/aravindo/Downloads\a\r\e[0m\e[27m\e[24m\e[J❯ \e[K\e[?2004h"
+  - delay: 300
+    content: o
+  - delay: 100
+    content: "\bos"
+  - delay: 100
+    content: x
+  - delay: 100
+    content: p
+  - delay: 100
+    content: h
+  - delay: 100
+    content: o
+  - delay: 100
+    content: t
+  - delay: 100
+    content: o
+  - delay: 100
+    content: s
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: e
+  - delay: 100
+    content: x
+  - delay: 100
+    content: p
+  - delay: 100
+    content: o
+  - delay: 100
+    content: r
+  - delay: 100
+    content: t
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: e
+  - delay: 100
+    content: x
+  - delay: 100
+    content: p
+  - delay: 100
+    content: o
+  - delay: 100
+    content: r
+  - delay: 100
+    content: t
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: b
+  - delay: 100
+    content: 'y'
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: d
+  - delay: 100
+    content: a
+  - delay: 100
+    content: t
+  - delay: 100
+    content: e
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: f
+  - delay: 100
+    content: r
+  - delay: 100
+    content: o
+  - delay: 100
+    content: m
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: d
+  - delay: 100
+    content: a
+  - delay: 100
+    content: t
+  - delay: 100
+    content: e
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: '2'
+  - delay: 100
+    content: '0'
+  - delay: 100
+    content: '2'
+  - delay: 100
+    content: '1'
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: '0'
+  - delay: 100
+    content: '1'
+  - delay: 100
+    content: '-'
+  - delay: 100
+    content: '0'
+  - delay: 100
+    content: '1'
+  - delay: 100
+    content: ' '
+  - delay: 100
+    content: t
+  - delay: 100
+    content: r
+  - delay: 100
+    content: i
+  - delay: 100
+    content: p
+  - delay: 300
+    content: "\e[?2004l\r\r\n"
+  - delay: 500
+    content: "Using last opened Photos library: /Users/user/Pictures/Photos Library.photoslibrary\r\n"
+  - delay: 8204
+    content: "Exporting 79 photos to /Users/user/trip...\r\n"
+  - delay: 321
+    content: "Processed: 79 photos, exported: 80, missing: 0, error: 0\r\nElapsed time: 0.321 seconds\r\n"
+  - delay: 317
+    content: "\e[1m\e[7m%\e[27m\e[1m\e[0m                                                                                          \r \r\e]7;file://wingeier-macOS/Users/aravindo/Downloads\a\r\e[0m\e[27m\e[24m\e[J❯ \e[K\e[?2004h"
+  
+  - delay: 4252
+    content: "\e[7mdu -h trip\e[27m"
+  - delay: 487
+    content: "\e[10D\e[27md\e[27mu\e[27m \e[27m-\e[27mh\e[27m \e[27mt\e[27mr\e[27mi\e[27mp\e[?2004l\r\r\n"
+  - delay: 7
+    content: "229M\ttrip/2021/01/03\r\n712K\ttrip/2021/01/02\r\n7.5M\ttrip/2021/01/01\r\n237M\ttrip/2021/01\r\n237M\ttrip/2021\r\n238M\ttrip\r\n\e[1m\e[7m%\e[27m\e[1m\e[0m                                                                                          \r \r\e]7;file://wingeier-macOS/Users/aravindo/Downloads\a\r\e[0m\e[27m\e[24m\e[J❯ \e[K\e[?2004h"
+  - delay: 4280
+    content: "\e[7mfind trip | head -20\e[27m"
+  - delay: 923
+    content: "\e[20D\e[27mf\e[27mi\e[27mn\e[27md\e[27m \e[27mt\e[27mr\e[27mi\e[27mp\e[27m \e[27m|\e[27m \e[27mh\e[27me\e[27ma\e[27md\e[27m \e[27m-\e[27m2\e[27m0\e[?2004l\r\r\n"
+  - delay: 5
+    content: "trip\r\ntrip/2021\r\ntrip/2021/01\r\ntrip/2021/01/03\r\ntrip/2021/01/03/IMG_1234 (1).HEIC\r\ntrip/2021/01/03/IMG_1267.HEIC\r\ntrip/2021/01/03/IMG_1226.HEIC\r\ntrip/2021/01/03/IMG_1271.HEIC\r\ntrip/2021/01/03/IMG_1232 (1).JPG\r\ntrip/2021/01/03/IMG_1270.HEIC\r\ntrip/2021/01/03/IMG_1231.HEIC\r\ntrip/2021/01/03/IMG_6926.JPG\r\ntrip/2021/01/03/IMG_6932.JPG\r\ntrip/2021/01/03/IMG_1266.HEIC\r\ntrip/2021/01/03/IMG_6933.JPG\r\ntrip/2021/01/03/IMG_6927.JPG\r\ntrip/2021/01/03/IMG_1233 (1).JPG\r\ntrip/2021/01/03/IMG_1228 (1).HEIC\r\ntrip/2021/01/03/IMG_6931.JPG\r\ntrip/2021/01/03/IMG_6930.JPG\r\n\e[1m\e[7m%\e[27m\e[1m\e[0m                                                                                          \r \r\e]7;file://wingeier-macOS/Users/aravindo/Downloads\a\r\e[0m\e[27m\e[24m\e[J❯ \e[K\e[?2004h"
+  - delay: 3615
+    content: "\e[?2004l\r\r\n"  

examples/export_by_album.py

@@ -79,7 +79,7 @@ def export(export_path, default_album, library_path, edited):
                 exported = p.export(dest_dir, filename)
                 click.echo(f"Exported {filename} to {exported}")
         else:
-            click.echo(f"Skipping missing photo: {p.original_filename} in album {album}")
+            click.echo(f"Skipping missing photo: {p.original_filename}")
 
 
 if __name__ == "__main__":

examples/export_faces.py

@@ -0,0 +1,83 @@
+""" Export all photos that contain a detected face and draw rectangles around each face
+    photos with no persons/detected faces will not be export 
+
+    This shows how to use the FaceInfo class and is useful for validating that FaceInfo is
+    correctly handling faces.
+
+    To use this, you'll need to install Pillow:
+    python3 -m pip install Pillow
+"""
+
+import os
+
+import click
+from PIL import Image, ImageDraw
+
+import osxphotos
+
+
+@click.command()
+@click.argument("export-path", type=click.Path(exists=True))
+@click.option(
+    "--uuid",
+    metavar="UUID",
+    help="Limit export to optional UUID(s)",
+    required=False,
+    multiple=True,
+)
+@click.option(
+    "--library-path",
+    metavar="PATH",
+    help="Path to Photos library, default to last used library",
+    default=None,
+)
+def export(export_path, library_path, uuid):
+    """ export photos to export_path and draw faces """
+    library_path = os.path.expanduser(library_path) if library_path else None
+    if library_path is not None:
+        photosdb = osxphotos.PhotosDB(library_path)
+    else:
+        photosdb = osxphotos.PhotosDB()
+
+    photos = photosdb.photos(uuid=uuid) if uuid else photosdb.photos(movies=False)
+    for p in photos:
+        if p.person_info and not p.ismissing:
+            # has persons and not missing
+            if "heic" in p.filename.lower():
+                print(f"skipping heic image {p.filename}")
+                continue
+            print(f"exporting photo {p.original_filename}, uuid = {p.uuid}")
+            export = p.export(export_path, p.original_filename, edited=p.hasadjustments)
+            if export:
+                im = Image.open(export[0])
+                draw = ImageDraw.Draw(im)
+                for face in p.face_info:
+                    coords = face.face_rect()
+                    draw.rectangle(coords, width=3)
+                    draw.ellipse(get_circle_points(face.center, 3), width=1)
+                    draw.text(face.mouth, "M", fill=(255, 255, 255, 255))
+                    draw.text(face.left_eye, "L", fill=(255, 255, 255, 255))
+                    draw.text(face.right_eye, "R", fill=(255, 255, 255, 255))
+                im.save(export[0])
+            else:
+                print(f"no photos exported for {p.uuid}")
+
+
+def get_circle_points(xy, radius):
+    """ Returns tuples of (x0, y0), (x1, y1) for a circle centered at x, y with radius
+
+    Arguments:
+        xy: tuple of x, y coordinates
+        radius: radius of circle to draw
+
+    Returns:
+        [(x0, y0), (x1, y1)] for bounding box of circle centered at x, y
+    """
+    x, y = xy
+    x0, y0 = x - radius, y - radius
+    x1, y1 = x + radius, y + radius
+    return [(x0, y0), (x1, y1)]
+
+
+if __name__ == "__main__":
+    export()  # pylint: disable=no-value-for-parameter

examples/force_download.py

@@ -0,0 +1,42 @@
+""" use osxphotos to force the download of photos from iCloud
+    downloads images to a temporary directory then deletes them
+    resulting in the photo being downloaded to Photos library
+"""
+
+import os
+import sys
+import tempfile
+
+import osxphotos
+
+
+def main():
+    photosdb = osxphotos.PhotosDB()
+    tempdir = tempfile.TemporaryDirectory()
+    photos = photosdb.photos()
+    downloaded = 0
+    missing = [photo for photo in photos if photo.ismissing and not photo.shared]
+
+    if not missing:
+        print(f"Did not find any missing photos to download")
+        sys.exit(0)
+
+    print(f"Downloading {len(missing)} photos")
+    for photo in missing:
+        if photo.ismissing:
+            print(f"Downloading photo {photo.original_filename}")
+            downloaded += 1
+            exported = photo.export(tempdir.name, use_photos_export=True, timeout=300)
+            if photo.hasadjustments:
+                exported.extend(
+                    photo.export(tempdir.name, use_photos_export=True, edited=True, timeout=300)
+                )
+            for filename in exported:
+                print(f"Removing temporary file {filename}")
+                os.unlink(filename)
+    print(f"Downloaded {downloaded} photos")
+    tempdir.cleanup()
+
+
+if __name__ == "__main__":
+    main()

examples/get_shared_photo_comments.py

@@ -0,0 +1,156 @@
+""" get shared comments associated with a photo """
+
+import datetime
+import sys
+from dataclasses import dataclass
+
+import osxphotos
+from osxphotos._constants import TIME_DELTA
+
+
+@dataclass
+class Comment:
+    """ Class for shared photo comments """
+
+    uuid: str
+    sort_fok: int
+    datetime: datetime.datetime
+    user: str
+    ismine: bool
+    text: str
+
+
+@dataclass
+class Like:
+    """ Class for shared photo likes """
+
+    uuid: str
+    sort_fok: int
+    datetime: datetime.datetime
+    user: str
+    ismine: bool
+
+
+def get_shared_person_info(photosdb, hashed_person_id):
+    """ returns tuple of (first name, last name, full name) 
+        for person invited to shared album with 
+        ZINVITEEHASHEDPERSONID = hashed_person_id
+        
+    Args:
+        photosdb: a osxphotos.PhotosDB object
+        hashed_person_id: str, value of ZINVITEEHASHEDPERSONID to lookup
+    """
+
+    conn, _ = photosdb.get_db_connection()
+    results = conn.execute(
+        """
+        SELECT
+        ZINVITEEHASHEDPERSONID,
+        ZINVITEEFIRSTNAME,
+        ZINVITEELASTNAME,
+        ZINVITEEFULLNAME
+        FROM
+        ZCLOUDSHAREDALBUMINVITATIONRECORD
+        WHERE
+        ZINVITEEHASHEDPERSONID = ?
+        LIMIT 1
+        """,
+        ([hashed_person_id]),
+    ).fetchall()
+
+    if results:
+        row = results[0]
+        return (row[1], row[2], row[3])
+    else:
+        return (None, None, None)
+
+
+def get_comments(photosdb, uuid):
+    """ return comments and likes, if any, for photo with uuid
+
+    Args:
+        photosdb: a osxphotos.PhotosDB object
+        uuid: uuid of the photo
+    
+    Returns:
+        tuple of (list of comments as Comment objects or [] if no comments, list of likes as Like objects or [] if no likes)
+    """
+    conn, _ = photosdb.get_db_connection()
+
+    results = conn.execute(
+        """
+        SELECT 
+        ZGENERICASSET.ZUUID, --0: UUID of the photo
+        ZCLOUDSHAREDCOMMENT.ZISLIKE, --1: comment is actually a "like"
+        ZCLOUDSHAREDCOMMENT.Z_FOK_COMMENTEDASSET, --2: sort order for comments on a photo
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTDATE, --3: date of comment
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTTEXT, --4: text of comment
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTERHASHEDPERSONID, --5: hashed ID of person who made comment/like
+        ZCLOUDSHAREDCOMMENT.ZISMYCOMMENT --6: is my (this user's) comment
+        FROM ZCLOUDSHAREDCOMMENT
+        JOIN ZGENERICASSET ON
+        ZGENERICASSET.Z_PK = ZCLOUDSHAREDCOMMENT.ZCOMMENTEDASSET
+        OR
+        ZGENERICASSET.Z_PK = ZCLOUDSHAREDCOMMENT.ZLIKEDASSET
+        WHERE ZGENERICASSET.ZUUID = ?
+    """,
+        ([uuid]),
+    ).fetchall()
+
+    comments = []
+    likes = []
+    for row in results:
+        photo_uuid = row[0]
+        sort_fok = row[2] or 0  # sort_fok is Null/None for likes
+        is_like = bool(row[1])
+        text = row[4]
+        user_info = get_shared_person_info(photosdb, row[5])
+        try:
+            dt = datetime.datetime.fromtimestamp(row[3] + TIME_DELTA)
+        except:
+            dt = datetime.datetime(1970, 1, 1)
+        ismine = bool(row[6])
+        if is_like:
+            # it's a like
+            likes.append(Like(photo_uuid, sort_fok, dt, user_info[2], ismine))
+        elif text:
+            # comment
+            comments.append(
+                Comment(photo_uuid, sort_fok, dt, user_info[2], ismine, text)
+            )
+    if likes:
+        likes.sort(key=lambda x: x.datetime)
+    if comments:
+        comments.sort(key=lambda x: x.sort_fok)
+    return (comments, likes)
+
+
+def main():
+    if len(sys.argv) > 1:
+        # library as first argument
+        photosdb = osxphotos.PhotosDB(dbfile=sys.argv[1])
+    else:
+        # open default library
+        photosdb = osxphotos.PhotosDB()
+
+    # shared albums
+    shared_albums = photosdb.album_info_shared
+    for album in shared_albums:
+        print(f"Processing album {album.title}")
+        # only shared albums can have comments
+        for photo in album.photos:
+            comments, likes = get_comments(photosdb, photo.uuid)
+            if comments or likes:
+                print(f"{photo.uuid}, {photo.original_filename}: ")
+            if likes:
+                print("Likes:")
+                for like in likes:
+                    print(like)
+            if comments:
+                print("Comments:")
+                for comment in comments:
+                    print(comment)
+
+
+if __name__ == "__main__":
+    main()

examples/photos_repl.py

@@ -7,6 +7,7 @@
 # If you run this using python from command line, do so with -i flag:
 # python3 -i examples/photos_repl.py
 
+import os
 import sys
 import time
 
@@ -17,18 +18,31 @@ import osxphotos
 from osxphotos.__main__ import get_photos_db, _list_libraries
 
 
+def show(photo):
+    """ open image with default image viewer
+    
+    Note: This is for debugging only -- it will actually open any filetype which could 
+    be very, very bad.  
+
+    Args:
+        photo: PhotoInfo object or a path to a photo on disk
+    """
+    photopath = photo.path if isinstance(photo, osxphotos.PhotoInfo) else photo
+
+    if not os.path.isfile(photopath):
+        return f"'{photopath}' does not appear to be a valid photo path"
+
+    os.system(f"open '{photopath}'")
+
+
 def main():
     db = None
 
-    if len(sys.argv) > 1:
-        db = sys.argv[1]
-    else:
-        db = get_photos_db()
-
+    db = sys.argv[1] if len(sys.argv) > 1 else get_photos_db()
     if db:
         print("loading database")
         tic = time.perf_counter()
-        photosdb = osxphotos.PhotosDB(dbfile=db)
+        photosdb = osxphotos.PhotosDB(dbfile=db, verbose=print)
         toc = time.perf_counter()
         print(f"done: took {toc-tic} seconds")
         return photosdb
@@ -44,5 +58,6 @@ if __name__ == "__main__":
     print("getting photos")
     tic = time.perf_counter()
     photos = photosdb.photos(images=True, movies=True)
+    photos.extend(photosdb.photos(images=True, movies=True, intrash=True))
     toc = time.perf_counter()
     print(f"found {len(photos)} photos in {toc-tic} seconds")

make_cli_exe.sh

@@ -5,4 +5,4 @@
 # If you need to install pyinstaller:
 # python3 -m pip install --upgrade pyinstaller
 
-pyinstaller --onefile --hidden-import="pkg_resources.py2_warn" --name osxphotos --add-data osxphotos/templates/xmp_sidecar.mako:osxphotos/templates cli.py
+pyinstaller osxphotos.spec

osxphotos.spec

@@ -0,0 +1,48 @@
+# -*- mode: python ; coding: utf-8 -*-
+# spec file for pyinstaller
+# run `pyinstaller osxphotos.spec`
+
+import os
+import importlib
+
+pathex = os.getcwd()
+
+# include necessary data files
+datas=[('osxphotos/templates/xmp_sidecar.mako', 'osxphotos/templates'), ('osxphotos/templates/xmp_sidecar_beta.mako', 'osxphotos/templates')]
+package_imports = [['photoscript', ['photoscript.applescript']]]
+for package, files in package_imports:
+    proot = os.path.dirname(importlib.import_module(package).__file__)
+    datas.extend((os.path.join(proot, f), package) for f in files)
+
+block_cipher = None
+
+a = Analysis(['cli.py'],
+             pathex=[pathex],
+             binaries=[],
+             datas=datas,
+             hiddenimports=['pkg_resources.py2_warn'],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [],
+          name='osxphotos',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True )

osxphotos/__init__.py

@@ -1,16 +1,10 @@
-import logging
-
 from ._version import __version__
 from .photoinfo import PhotoInfo
 from .photosdb import PhotosDB
-from .utils import _set_debug, _debug, _get_logger
+from .photosdb._photosdb_process_comments import CommentInfo, LikeInfo
+from .phototemplate import PhotoTemplate
+from .utils import _debug, _get_logger, _set_debug
 
-# TODO: find edited photos: see https://github.com/orangeturtle739/photos-export/blob/master/extract_photos.py
 # TODO: Add test for imageTimeZoneOffsetSeconds = None
-# TODO: Fix command line so multiple --keyword, etc. are AND (instead of OR as they are in .photos())
-#       Or fix the help text to match behavior
 # TODO: Add test for __str__ and to_json
-# TODO: fix docstrings
 # TODO: Add special albums and magic albums
-# TODO: cleanup os.path and pathlib code (import pathlib and also from pathlib import Path)
-

osxphotos/_constants.py

@@ -3,6 +3,16 @@ Constants used by osxphotos
 """
 
 import os.path
+from datetime import datetime
+
+OSXPHOTOS_URL = "https://github.com/RhetTbull/osxphotos"
+
+# Time delta: add this to Photos times to get unix time
+# Apple Epoch is Jan 1, 2001
+TIME_DELTA = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
+
+# Unicode format to use for comparing strings
+UNICODE_FORMAT = "NFC"
 
 # which Photos library database versions have been tested
 # Photos 2.0 (10.12.6) == 2622
@@ -10,18 +20,56 @@ import os.path
 # Photos 4.0 (10.14.5) == 4016
 # Photos 4.0 (10.14.6) == 4025
 # Photos 5.0 (10.15.0) == 6000
-# TODO: Should this also use compatibleBackToVersion from LiGlobals?
 _TESTED_DB_VERSIONS = ["6000", "4025", "4016", "3301", "2622"]
 
+# database model versions (applies to Photos 5, Photos 6)
+# these come from PLModelVersion key in binary plist in Z_METADATA.Z_PLIST
+# Photos 5 (10.15.1) == 13537
+# Photos 5 (10.15.4, 10.15.5, 10.15.6) == 13703
+# Photos 6 (10.16.0 Beta) == 14104
+_TEST_MODEL_VERSIONS = ["13537", "13703", "14104"]
+
 # only version 3 - 4 have RKVersion.selfPortrait
 _PHOTOS_3_VERSION = "3301"
 
 # versions 5.0 and later have a different database structure
 _PHOTOS_4_VERSION = "4025"  # latest Mojove version on 10.14.6
-_PHOTOS_5_VERSION = "6000"  # seems to be current on 10.15.1 through 10.15.4
+_PHOTOS_5_VERSION = "6000"  # seems to be current on 10.15.1 through 10.15.6
 
-# which major version operating systems have been tested
-_TESTED_OS_VERSIONS = ["12", "13", "14", "15"]
+# Ranges for model version by Photos version
+_PHOTOS_5_MODEL_VERSION = [13000, 13999]
+_PHOTOS_6_MODEL_VERSION = [14000, 14999]
+
+# some table names differ between Photos 5 and Photos 6
+_DB_TABLE_NAMES = {
+    5: {
+        "ASSET": "ZGENERICASSET",
+        "KEYWORD_JOIN": "Z_1KEYWORDS.Z_37KEYWORDS",
+        "ALBUM_JOIN": "Z_26ASSETS.Z_34ASSETS",
+        "ALBUM_SORT_ORDER": "Z_26ASSETS.Z_FOK_34ASSETS",
+        "IMPORT_FOK": "ZGENERICASSET.Z_FOK_IMPORTSESSION",
+        "DEPTH_STATE": "ZGENERICASSET.ZDEPTHSTATES",
+    },
+    6: {
+        "ASSET": "ZASSET",
+        "KEYWORD_JOIN": "Z_1KEYWORDS.Z_36KEYWORDS",
+        "ALBUM_JOIN": "Z_26ASSETS.Z_3ASSETS",
+        "ALBUM_SORT_ORDER": "Z_26ASSETS.Z_FOK_3ASSETS",
+        "IMPORT_FOK": "null",
+        "DEPTH_STATE": "ZASSET.ZDEPTHTYPE",
+    },
+}
+
+# which version operating systems have been tested
+_TESTED_OS_VERSIONS = [
+    ("10", "12"),
+    ("10", "13"),
+    ("10", "14"),
+    ("10", "15"),
+    ("10", "16"),
+    ("11", "0"),
+    ("11", "1"),
+]
 
 # Photos 5 has persons who are empty string if unidentified face
 _UNKNOWN_PERSON = "_UNKNOWN_"
@@ -41,12 +89,14 @@ _MOVIE_TYPE = 1
 # Name of XMP template file
 _TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates")
 _XMP_TEMPLATE_NAME = "xmp_sidecar.mako"
+_XMP_TEMPLATE_NAME_BETA = "xmp_sidecar_beta.mako"
 
 # Constants used for processing folders and albums
 _PHOTOS_5_ALBUM_KIND = 2  # normal user album
 _PHOTOS_5_SHARED_ALBUM_KIND = 1505  # shared album
 _PHOTOS_5_FOLDER_KIND = 4000  # user folder
 _PHOTOS_5_ROOT_FOLDER_KIND = 3999  # root folder
+_PHOTOS_5_IMPORT_SESSION_ALBUM_KIND = 1506  # import session
 
 _PHOTOS_4_ALBUM_KIND = 3  # RKAlbum.albumSubclass
 _PHOTOS_4_TOP_LEVEL_ALBUM = "TopLevelAlbums"
@@ -63,3 +113,96 @@ _OSXPHOTOS_NONE_SENTINEL = "OSXPhotosXYZZY42_Sentinel$"
 
 # SearchInfo categories for Photos 5, corresponds to categories in database/search/psi.sqlite
 SEARCH_CATEGORY_LABEL = 2024
+SEARCH_CATEGORY_PLACE_NAME = 1
+SEARCH_CATEGORY_STREET = 2
+SEARCH_CATEGORY_NEIGHBORHOOD = 3
+SEARCH_CATEGORY_LOCALITY_4 = 4
+SEARCH_CATEGORY_SUB_LOCALITY_5 = 5
+SEARCH_CATEGORY_SUB_LOCALITY_6 = 6
+SEARCH_CATEGORY_CITY = 7
+SEARCH_CATEGORY_LOCALITY_8 = 8
+SEARCH_CATEGORY_NAMED_AREA = 9
+SEARCH_CATEGORY_ALL_LOCALITY = [
+    SEARCH_CATEGORY_LOCALITY_4,
+    SEARCH_CATEGORY_SUB_LOCALITY_5,
+    SEARCH_CATEGORY_SUB_LOCALITY_6,
+    SEARCH_CATEGORY_LOCALITY_8,
+    SEARCH_CATEGORY_NAMED_AREA,
+]
+SEARCH_CATEGORY_STATE = 10
+SEARCH_CATEGORY_STATE_ABBREVIATION = 11
+SEARCH_CATEGORY_COUNTRY = 12
+SEARCH_CATEGORY_BODY_OF_WATER = 14
+SEARCH_CATEGORY_MONTH = 1014
+SEARCH_CATEGORY_YEAR = 1015
+SEARCH_CATEGORY_KEYWORDS = 2016
+SEARCH_CATEGORY_TITLE = 2017
+SEARCH_CATEGORY_DESCRIPTION = 2018
+SEARCH_CATEGORY_HOME = 2020
+SEARCH_CATEGORY_PERSON = 2021
+SEARCH_CATEGORY_ACTIVITY = 2027
+SEARCH_CATEGORY_HOLIDAY = 2029
+SEARCH_CATEGORY_SEASON = 2030
+SEARCH_CATEGORY_WORK = 2036
+SEARCH_CATEGORY_VENUE = 2038
+SEARCH_CATEGORY_VENUE_TYPE = 2039
+SEARCH_CATEGORY_PHOTO_TYPE_VIDEO = 2044
+SEARCH_CATEGORY_PHOTO_TYPE_SLOMO = 2045
+SEARCH_CATEGORY_PHOTO_TYPE_LIVE = 2046
+SEARCH_CATEGORY_PHOTO_TYPE_SCREENSHOT = 2047
+SEARCH_CATEGORY_PHOTO_TYPE_PANORAMA = 2048
+SEARCH_CATEGORY_PHOTO_TYPE_TIMELAPSE = 2049
+SEARCH_CATEGORY_PHOTO_TYPE_BURSTS = 2052
+SEARCH_CATEGORY_PHOTO_TYPE_PORTRAIT = 2053
+SEARCH_CATEGORY_PHOTO_TYPE_SELFIES = 2054
+SEARCH_CATEGORY_PHOTO_TYPE_FAVORITES = 2055
+SEARCH_CATEGORY_MEDIA_TYPES = [
+    SEARCH_CATEGORY_PHOTO_TYPE_VIDEO,
+    SEARCH_CATEGORY_PHOTO_TYPE_SLOMO,
+    SEARCH_CATEGORY_PHOTO_TYPE_LIVE,
+    SEARCH_CATEGORY_PHOTO_TYPE_SCREENSHOT,
+    SEARCH_CATEGORY_PHOTO_TYPE_PANORAMA,
+    SEARCH_CATEGORY_PHOTO_TYPE_TIMELAPSE,
+    SEARCH_CATEGORY_PHOTO_TYPE_BURSTS,
+    SEARCH_CATEGORY_PHOTO_TYPE_PORTRAIT,
+    SEARCH_CATEGORY_PHOTO_TYPE_SELFIES,
+    SEARCH_CATEGORY_PHOTO_TYPE_FAVORITES,
+]
+SEARCH_CATEGORY_PHOTO_NAME = 2056
+
+
+# Max filename length on MacOS
+MAX_FILENAME_LEN = 255
+
+# Max directory name length on MacOS
+MAX_DIRNAME_LEN = 255
+
+# Default JPEG quality when converting to JPEG
+DEFAULT_JPEG_QUALITY = 1.0
+
+# Default suffix to add to edited images
+DEFAULT_EDITED_SUFFIX = "_edited"
+
+# Default suffix to add to original images
+DEFAULT_ORIGINAL_SUFFIX = ""
+
+# Colors for print CLI messages
+CLI_COLOR_ERROR = "red"
+CLI_COLOR_WARNING = "yellow"
+
+# Bit masks for --sidecar
+SIDECAR_JSON = 0x1
+SIDECAR_EXIFTOOL = 0x2
+SIDECAR_XMP = 0x4
+
+# supported attributes for --xattr-template
+EXTENDED_ATTRIBUTE_NAMES = [
+    "authors",
+    "comment",
+    "copyright",
+    "description",
+    "findercomment",
+    "headline",
+    "keywords",
+]
+EXTENDED_ATTRIBUTE_NAMES_QUOTED = [f"'{x}'" for x in EXTENDED_ATTRIBUTE_NAMES]

osxphotos/_export_db.py

@@ -1,439 +0,0 @@
-""" Helper class for managing a database used by 
-    PhotoInfo.export for tracking state of exports and updates
-"""
-
-import datetime
-import logging
-import os
-import pathlib
-import sqlite3
-import sys
-from abc import ABC, abstractmethod
-from sqlite3 import Error
-
-from ._version import __version__
-
-OSXPHOTOS_EXPORTDB_VERSION = "1.0"
-
-
-class ExportDB_ABC(ABC):
-    @abstractmethod
-    def get_uuid_for_file(self, filename):
-        pass
-
-    @abstractmethod
-    def set_uuid_for_file(self, filename, uuid):
-        pass
-
-    @abstractmethod
-    def set_stat_orig_for_file(self, filename, stats):
-        pass
-
-    @abstractmethod
-    def get_stat_orig_for_file(self, filename):
-        pass
-
-    @abstractmethod
-    def set_stat_exif_for_file(self, filename, stats):
-        pass
-
-    @abstractmethod
-    def get_stat_exif_for_file(self, filename):
-        pass
-
-    @abstractmethod
-    def get_info_for_uuid(self, uuid):
-        pass
-
-    @abstractmethod
-    def set_info_for_uuid(self, uuid, info):
-        pass
-
-    @abstractmethod
-    def get_exifdata_for_file(self, uuid):
-        pass
-
-    @abstractmethod
-    def set_exifdata_for_file(self, uuid, exifdata):
-        pass
-
-    @abstractmethod
-    def set_data(self, file, uuid, orig_stat, exif_stat, info_json, exif_json):
-        pass
-
-
-class ExportDBNoOp(ExportDB_ABC):
-    """ An ExportDB with NoOp methods """
-
-    def get_uuid_for_file(self, filename):
-        pass
-
-    def set_uuid_for_file(self, filename, uuid):
-        pass
-
-    def set_stat_orig_for_file(self, filename, stats):
-        pass
-
-    def get_stat_orig_for_file(self, filename):
-        pass
-
-    def set_stat_exif_for_file(self, filename, stats):
-        pass
-
-    def get_stat_exif_for_file(self, filename):
-        pass
-
-    def get_info_for_uuid(self, uuid):
-        pass
-
-    def set_info_for_uuid(self, uuid, info):
-        pass
-
-    def get_exifdata_for_file(self, uuid):
-        pass
-
-    def set_exifdata_for_file(self, uuid, exifdata):
-        pass
-
-    def set_data(self, file, uuid, orig_stat, exif_stat, info_json, exif_json):
-        pass
-
-
-class ExportDB(ExportDB_ABC):
-    """ Interface to sqlite3 database used to store state information for osxphotos export command """
-
-    def __init__(self, dbfile):
-        """ dbfile: path to osxphotos export database file """
-        self._dbfile = dbfile
-        # _path is parent of the database
-        # all files referenced by get_/set_uuid_for_file will be converted to
-        # relative paths to this parent _path
-        # this allows the entire export tree to be moved to a new disk/location
-        # whilst preserving the UUID to filename mappping
-        self._path = pathlib.Path(dbfile).parent
-        self._conn = self._open_export_db(dbfile)
-        self._insert_run_info()
-
-    def get_uuid_for_file(self, filename):
-        """ query database for filename and return UUID
-            returns None if filename not found in database
-        """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        logging.debug(f"get_uuid: {filename}")
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                f"SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,)
-            )
-            results = c.fetchone()
-            uuid = results[0] if results else None
-        except Error as e:
-            logging.warning(e)
-            uuid = None
-
-        logging.debug(f"get_uuid: {uuid}")
-        return uuid
-
-    def set_uuid_for_file(self, filename, uuid):
-        """ set UUID of filename to uuid in the database """
-        filename = str(pathlib.Path(filename).relative_to(self._path))
-        filename_normalized = filename.lower()
-        logging.debug(f"set_uuid: {filename} {uuid}")
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);",
-                (filename, filename_normalized, uuid),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-    def set_stat_orig_for_file(self, filename, stats):
-        """ set stat info for filename
-            filename: filename to set the stat info for
-            stat: a tuple of length 3: mode, size, mtime """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        if len(stats) != 3:
-            raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
-
-        logging.debug(f"set_stat_orig_for_file: {filename} {stats}")
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "UPDATE files "
-                + "SET orig_mode = ?, orig_size = ?, orig_mtime = ? "
-                + "WHERE filepath_normalized = ?;",
-                (*stats, filename),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-    def get_stat_orig_for_file(self, filename):
-        """ get stat info for filename
-            returns: tuple of (mode, size, mtime) 
-        """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "SELECT orig_mode, orig_size, orig_mtime FROM files WHERE filepath_normalized = ?",
-                (filename,),
-            )
-            results = c.fetchone()
-            stats = results[0:3] if results else None
-        except Error as e:
-            logging.warning(e)
-            stats = (None, None, None)
-
-        logging.debug(f"get_stat_orig_for_file: {stats}")
-        return stats
-
-    def set_stat_exif_for_file(self, filename, stats):
-        """ set stat info for filename (after exiftool has updated it)
-            filename: filename to set the stat info for
-            stat: a tuple of length 3: mode, size, mtime """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        if len(stats) != 3:
-            raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
-
-        logging.debug(f"set_stat_exif_for_file: {filename} {stats}")
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "UPDATE files "
-                + "SET exif_mode = ?, exif_size = ?, exif_mtime = ? "
-                + "WHERE filepath_normalized = ?;",
-                (*stats, filename),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-    def get_stat_exif_for_file(self, filename):
-        """ get stat info for filename (after exiftool has updated it)
-            returns: tuple of (mode, size, mtime) 
-        """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "SELECT exif_mode, exif_size, exif_mtime FROM files WHERE filepath_normalized = ?",
-                (filename,),
-            )
-            results = c.fetchone()
-            stats = results[0:3] if results else None
-        except Error as e:
-            logging.warning(e)
-            stats = (None, None, None)
-
-        logging.debug(f"get_stat_exif_for_file: {stats}")
-        return stats
-
-    def get_info_for_uuid(self, uuid):
-        """ returns the info JSON struct for a UUID """
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute("SELECT json_info FROM info WHERE uuid = ?", (uuid,))
-            results = c.fetchone()
-            info = results[0] if results else None
-        except Error as e:
-            logging.warning(e)
-            info = None
-
-        logging.debug(f"get_info: {uuid}, {info}")
-        return info
-
-    def set_info_for_uuid(self, uuid, info):
-        """ sets the info JSON struct for a UUID """
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "INSERT OR REPLACE INTO info(uuid, json_info) VALUES (?, ?);",
-                (uuid, info),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-        logging.debug(f"set_info: {uuid}, {info}")
-
-    def get_exifdata_for_file(self, filename):
-        """ returns the exifdata JSON struct for a file """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "SELECT json_exifdata FROM exifdata WHERE filepath_normalized = ?",
-                (filename,),
-            )
-            results = c.fetchone()
-            exifdata = results[0] if results else None
-        except Error as e:
-            logging.warning(e)
-            exifdata = None
-
-        logging.debug(f"get_exifdata: {filename}, {exifdata}")
-        return exifdata
-
-    def set_exifdata_for_file(self, filename, exifdata):
-        """ sets the exifdata JSON struct for a file """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                "INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
-                (filename, exifdata),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-        logging.debug(f"set_exifdata: {filename}, {exifdata}")
-
-    def set_data(self, file, uuid, orig_stat, exif_stat, info_json, exif_json):
-        """ sets all the data for file and uuid at once 
-            calls set_uuid_for_file
-                  set_info_for_uuid
-                  set_stat_orig_for_file
-                  set_stat_exif_for_file
-                  set_exifdata_for_file        
-        """
-        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
-
-        self.set_uuid_for_file(filename, uuid)
-        self.set_info_for_uuid(uuid, info_json)
-        self.set_stat_orig_for_file(filename, orig_stat)
-        self.set_stat_exif_for_file(filename, exif_stat)
-        self.set_exifdata_for_file(filename, exif_json)
-
-    def close(self):
-        """ close the database connection """
-        try:
-            self._conn.close()
-        except Error as e:
-            logging.warning(e)
-
-    def _open_export_db(self, dbfile):
-        """ open export database and return a db connection
-            if dbfile does not exist, will create and initialize the database 
-            returns: connection to the database 
-        """
-
-        if not os.path.isfile(dbfile):
-            logging.debug(f"dbfile {dbfile} doesn't exist, creating it")
-            conn = self._get_db_connection(dbfile)
-            if conn:
-                self._create_db_tables(conn)
-            else:
-                raise Exception("Error getting connection to database {dbfile}")
-        else:
-            logging.debug(f"dbfile {dbfile} exists, opening it")
-            conn = self._get_db_connection(dbfile)
-
-        return conn
-
-    def _get_db_connection(self, dbfile):
-        """ return db connection to dbname """
-        try:
-            conn = sqlite3.connect(dbfile)
-        except Error as e:
-            logging.warning(e)
-            conn = None
-
-        return conn
-
-    def _create_db_tables(self, conn):
-        """ create (if not already created) the necessary db tables for the export database
-            conn: sqlite3 db connection 
-        """
-        sql_commands = {
-            "sql_version_table": """ CREATE TABLE IF NOT EXISTS version (
-                                id INTEGER PRIMARY KEY,
-                                osxphotos TEXT,
-                                exportdb TEXT 
-                                ); """,
-            "sql_files_table": """ CREATE TABLE IF NOT EXISTS files (
-                              id INTEGER PRIMARY KEY,
-                              filepath TEXT NOT NULL,
-                              filepath_normalized TEXT NOT NULL,
-                              uuid TEXT,
-                              orig_mode INTEGER,
-                              orig_size INTEGER,
-                              orig_mtime REAL,
-                              exif_mode INTEGER,
-                              exif_size INTEGER,
-                              exif_mtime REAL
-                              ); """,
-            "sql_runs_table": """ CREATE TABLE IF NOT EXISTS runs (
-                             id INTEGER PRIMARY KEY,
-                             datetime TEXT,
-                             python_path TEXT,
-                             script_name TEXT,
-                             args TEXT,
-                             cwd TEXT 
-                             ); """,
-            "sql_info_table": """ CREATE TABLE IF NOT EXISTS info (
-                             id INTEGER PRIMARY KEY,
-                             uuid text NOT NULL,
-                             json_info JSON 
-                             ); """,
-            "sql_exifdata_table": """ CREATE TABLE IF NOT EXISTS exifdata (
-                             id INTEGER PRIMARY KEY,
-                             filepath_normalized TEXT NOT NULL,
-                             json_exifdata JSON 
-                             ); """,
-            "sql_files_idx": """ CREATE UNIQUE INDEX idx_files_filepath_normalized on files (filepath_normalized); """,
-            "sql_info_idx": """ CREATE UNIQUE INDEX idx_info_uuid on info (uuid); """,
-            "sql_exifdata_idx": """ CREATE UNIQUE INDEX idx_exifdata_filename on exifdata (filepath_normalized); """,
-        }
-        try:
-            c = conn.cursor()
-            for cmd in sql_commands.values():
-                c.execute(cmd)
-            c.execute(
-                "INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);",
-                (__version__, OSXPHOTOS_EXPORTDB_VERSION),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)
-
-    def __del__(self):
-        """ ensure the database connection is closed """
-        if self._conn:
-            try:
-                self._conn.close()
-            except Error as e:
-                logging.warning(e)
-
-    def _insert_run_info(self):
-        dt = datetime.datetime.utcnow().isoformat()
-        python_path = sys.executable
-        cmd = sys.argv[0]
-        if len(sys.argv) > 1:
-            args = " ".join(sys.argv[1:])
-        else:
-            args = ""
-        cwd = os.getcwd()
-        conn = self._conn
-        try:
-            c = conn.cursor()
-            c.execute(
-                f"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)",
-                (dt, python_path, cmd, args, cwd),
-            )
-            conn.commit()
-        except Error as e:
-            logging.warning(e)

osxphotos/_filecmp.py

@@ -1,54 +0,0 @@
-"""Utilities for comparing files 
-
-Modified from CPython/Lib/filecmp.py
-
-Functions:
-    cmp_file(f1, s2) -> int
-    file_sig(f1) -> Tuple[int, int, float]
-
-"""
-
-import os
-import stat
-
-__all__ = ["cmp", "sig"]
-
-
-def cmp_file(f1, s2):
-    """Compare file f1 to signature s2.
-
-    Arguments:
-
-    f1 --  File name
-
-    s2 -- stats as returned by sig
-
-    Return value:
-
-    True if the files are the same, False otherwise.
-
-    This function uses a cache for past comparisons and the results,
-    with cache entries invalidated if their stat information
-    changes.  The cache may be cleared by calling clear_cache().
-
-    """
-
-    if not s2:
-        return False
-
-    s1 = _sig(os.stat(f1))
-
-    if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
-        return False
-    if s1 == s2:
-        return True
-    return False
-
-
-def _sig(st):
-    return (stat.S_IFMT(st.st_mode), st.st_size, st.st_mtime)
-
-
-def file_sig(f1):
-    """ return os.stat signature for file f1 """
-    return _sig(os.stat(f1))

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.29.2"
+__version__ = "0.40.2"

osxphotos/albuminfo.py

@@ -10,7 +10,7 @@ Represents a single Folder in the Photos library and provides access to the fold
 PhotosDB.folders() returns a list of FolderInfo objects
 """
 
-import logging
+from datetime import datetime, timedelta, timezone
 
 from ._constants import (
     _PHOTOS_4_ALBUM_KIND,
@@ -18,11 +18,34 @@ from ._constants import (
     _PHOTOS_4_VERSION,
     _PHOTOS_5_ALBUM_KIND,
     _PHOTOS_5_FOLDER_KIND,
+    TIME_DELTA,
 )
+from .datetime_utils import get_local_tz
 
 
-class AlbumInfo:
+def sort_list_by_keys(values, sort_keys):
+    """ Sorts list values by a second list sort_keys
+        e.g. given ["a","c","b"], [1, 3, 2], returns ["a", "b", "c"]
+
+    Args:
+        values: a list of values to be sorted
+        sort_keys: a list of keys to sort values by
+    
+    Returns:
+        list of values, sorted by sort_keys
+    
+    Raises:
+        ValueError: raised if len(values) != len(sort_keys)
     """
+    if len(values) != len(sort_keys):
+        return ValueError("values and sort_keys must have same length")
+
+    return list(zip(*sorted(zip(sort_keys, values))))[1]
+
+
+class AlbumInfoBaseClass:
+    """
+    Base class for AlbumInfo, ImportInfo
     Info about a specific Album, contains all the details about the album
     including folders, photos, etc.
     """
@@ -31,25 +54,111 @@ class AlbumInfo:
         self._uuid = uuid
         self._db = db
         self._title = self._db._dbalbum_details[uuid]["title"]
-
-    @property
-    def title(self):
-        """ return title / name of album """
-        return self._title
+        self._creation_date_timestamp = self._db._dbalbum_details[uuid]["creation_date"]
+        self._start_date_timestamp = self._db._dbalbum_details[uuid]["start_date"]
+        self._end_date_timestamp = self._db._dbalbum_details[uuid]["end_date"]
+        self._local_tz = get_local_tz(
+            datetime.fromtimestamp(self._creation_date_timestamp + TIME_DELTA)
+        )
 
     @property
     def uuid(self):
         """ return uuid of album """
         return self._uuid
 
+    @property
+    def creation_date(self):
+        """ return creation date of album """
+        try:
+            return self._creation_date
+        except AttributeError:
+            try:
+                self._creation_date = (
+                    datetime.fromtimestamp(
+                        self._creation_date_timestamp + TIME_DELTA
+                    ).astimezone(tz=self._local_tz)
+                    if self._creation_date_timestamp
+                    else datetime(1970, 1, 1, 0, 0, 0).astimezone(
+                        tz=timezone(timedelta(0))
+                    )
+                )
+            except ValueError:
+                self._creation_date = datetime(1970, 1, 1, 0, 0, 0).astimezone(
+                    tz=timezone(timedelta(0))
+                )
+            return self._creation_date
+
+    @property
+    def start_date(self):
+        """ For Albums, return start date (earliest image) of album or None for albums with no images
+            For Import Sessions, return start date of import session (when import began) """
+        try:
+            return self._start_date
+        except AttributeError:
+            try:
+                self._start_date = (
+                    datetime.fromtimestamp(
+                        self._start_date_timestamp + TIME_DELTA
+                    ).astimezone(tz=self._local_tz)
+                    if self._start_date_timestamp
+                    else None
+                )
+            except ValueError:
+                self._start_date = None
+            return self._start_date
+
+    @property
+    def end_date(self):
+        """ For Albums, return end date (most recent image) of album or None for albums with no images
+            For Import Sessions, return end date of import sessions (when import was completed) """
+        try:
+            return self._end_date
+        except AttributeError:
+            try:
+                self._end_date = (
+                    datetime.fromtimestamp(
+                        self._end_date_timestamp + TIME_DELTA
+                    ).astimezone(tz=self._local_tz)
+                    if self._end_date_timestamp
+                    else None
+                )
+            except ValueError:
+                self._end_date = None
+            return self._end_date
+
     @property
     def photos(self):
-        """ return list of photos contained in album """
+        return []
+
+    def __len__(self):
+        """ return number of photos contained in album """
+        return len(self.photos)
+
+
+class AlbumInfo(AlbumInfoBaseClass):
+    """
+    Base class for AlbumInfo, ImportInfo
+    Info about a specific Album, contains all the details about the album
+    including folders, photos, etc.
+    """
+
+    @property
+    def title(self):
+        """ return title / name of album """
+        return self._title
+
+    @property
+    def photos(self):
+        """ return list of photos contained in album sorted in same sort order as Photos """
         try:
             return self._photos
         except AttributeError:
-            uuid = self._db._dbalbums_album[self._uuid]
-            self._photos = self._db.photos(uuid=uuid)
+            if self.uuid in self._db._dbalbums_album:
+                uuid, sort_order = zip(*self._db._dbalbums_album[self.uuid])
+                sorted_uuid = sort_list_by_keys(uuid, sort_order)
+                self._photos = self._db.photos_by_uuid(sorted_uuid)
+            else:
+                self._photos = []
             return self._photos
 
     @property
@@ -62,10 +171,7 @@ class AlbumInfo:
         try:
             return self._folder_names
         except AttributeError:
-            if self._db._db_version <= _PHOTOS_4_VERSION:
-                self._folder_names = self._db._album_folder_hierarchy_list(self._uuid)
-            else:
-                self._folder_names = self._db._album_folder_hierarchy_list(self._uuid)
+            self._folder_names = self._db._album_folder_hierarchy_list(self._uuid)
             return self._folder_names
 
     @property
@@ -103,9 +209,24 @@ class AlbumInfo:
                 )
             return self._parent
 
-    def __len__(self):
-        """ return number of photos contained in album """
-        return len(self.photos)
+
+class ImportInfo(AlbumInfoBaseClass):
+    @property
+    def photos(self):
+        """ return list of photos contained in import session """
+        try:
+            return self._photos
+        except AttributeError:
+            uuid_list, sort_order = zip(
+                *[
+                    (uuid, self._db._dbphotos[uuid]["fok_import_session"])
+                    for uuid in self._db._dbphotos
+                    if self._db._dbphotos[uuid]["import_uuid"] == self.uuid
+                ]
+            )
+            sorted_uuid = sort_list_by_keys(uuid_list, sort_order)
+            self._photos = self._db.photos_by_uuid(sorted_uuid)
+            return self._photos
 
 
 class FolderInfo:

osxphotos/configoptions.py

@@ -0,0 +1,173 @@
+""" ConfigOptions class to load/save config settings for osxphotos CLI """
+import toml
+
+
+class ConfigOptionsException(Exception):
+    """ Invalid combination of options. """
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ConfigOptionsInvalidError(ConfigOptionsException):
+    pass
+
+
+class ConfigOptionsLoadError(ConfigOptionsException):
+    pass
+
+
+class ConfigOptions:
+    """ data class to store and load options for osxphotos commands """
+
+    def __init__(self, name, attrs, ignore=None):
+        """ init ConfigOptions class
+
+        Args:
+            name: name for these options, will be used for section heading in TOML file when saving/loading from file
+            attrs: dict with name and default value for all allowed attributes
+            ignore: optional list of strings of keys to ignore from attrs dict
+        """
+        self._name = name
+        self._attrs = attrs.copy()
+        if ignore:
+            for attrname in ignore:
+                self._attrs.pop(attrname, None)
+
+        self.set_attributes(attrs)
+
+    def set_attributes(self, args):
+        for attr in self._attrs:
+            try:
+                arg = args[attr]
+                # don't test 'not arg'; need to handle empty strings as valid values
+                if arg is None or arg == False:
+                    if type(self._attrs[attr]) == tuple:
+                        setattr(self, attr, ())
+                    else:
+                        setattr(self, attr, self._attrs[attr])
+                else:
+                    setattr(self, attr, arg)
+            except KeyError:
+                raise KeyError(f"Missing argument: {attr}")
+
+    def validate(self, exclusive=None, inclusive=None, dependent=None, cli=False):
+        """ validate combinations of otions
+        
+        Args:
+            exclusive: list of tuples in form [("option_1", "option_2")...] which are exclusive; 
+                       ie. either option_1 can be set or option_2 but not both; 
+            inclusive: list of tuples in form [("option_1", "option_2")...] which are inclusive; 
+                       ie. if either option_1 or option_2 is set, the other must be set
+            dependent: list of tuples in form [("option_1", ("option_2", "option_3"))...] 
+                       where if option_1 is set, then at least one of the options in the second tuple must also be set
+            cli: bool, set to True if called to validate CLI options; 
+                       will prepend '--' to option names in InvalidOptions.message and change _ to - in option names
+        
+        Returns:
+            True if all options valid
+            
+        Raises:
+            InvalidOption if any combination of options is invalid
+            InvalidOption.message will be descriptive message of invalid options
+        """
+        if not any([exclusive, inclusive, dependent]):
+            return True
+
+        prefix = "--" if cli else ""
+        if exclusive:
+            for a, b in exclusive:
+                vala = getattr(self, a)
+                valb = getattr(self, b)
+                vala = any(vala) if isinstance(vala, tuple) else vala
+                valb = any(valb) if isinstance(valb, tuple) else valb
+                if vala and valb:
+                    stra = a.replace("_", "-") if cli else a
+                    strb = b.replace("_", "-") if cli else b
+                    raise ConfigOptionsInvalidError(
+                        f"{prefix}{stra} and {prefix}{strb} options cannot be used together."
+                    )
+        if inclusive:
+            for a, b in inclusive:
+                vala = getattr(self, a)
+                valb = getattr(self, b)
+                vala = any(vala) if isinstance(vala, tuple) else vala
+                valb = any(valb) if isinstance(valb, tuple) else valb
+                if any([vala, valb]) and not all([vala, valb]):
+                    stra = a.replace("_", "-") if cli else a
+                    strb = b.replace("_", "-") if cli else b
+                    raise ConfigOptionsInvalidError(
+                        f"{prefix}{stra} and {prefix}{strb} options must be used together."
+                    )
+        if dependent:
+            for a, b in dependent:
+                vala = getattr(self, a)
+                if not isinstance(b, tuple):
+                    # python unrolls the tuple if there's a single element
+                    b = (b,)
+                valb = [getattr(self, x) for x in b]
+                valb = [any(x) if isinstance(x, tuple) else x for x in valb]
+                if vala and not any(valb):
+                    if cli:
+                        stra = prefix + a.replace("_", "-")
+                        strb = ", ".join(prefix + x.replace("_", "-") for x in b)
+                    else:
+                        stra = a
+                        strb = ", ".join(b)
+                    raise ConfigOptionsInvalidError(
+                        f"{stra} must be used with at least one of: {strb}."
+                    )
+        return True
+
+    def write_to_file(self, filename):
+        """ Write self to TOML file
+
+        Args:
+            filename: full path to TOML file to write; filename will be overwritten if it exists
+        """
+        # todo: add overwrite and option to merge contents already in TOML file (under different [section] with new content)
+        data = {}
+        for attr in sorted(self._attrs.keys()):
+            val = getattr(self, attr)
+            if val in [False, ()]:
+                val = None
+            else:
+                val = list(val) if type(val) == tuple else val
+
+            data[attr] = val
+
+        with open(filename, "w") as fd:
+            toml.dump({self._name: data}, fd)
+
+    def load_from_file(self, filename, override=False):
+        """ Load options from a TOML file.
+
+        Args:
+            filename: full path to TOML file
+            override: bool; if True, values in the TOML file will override values already set in the instance
+
+        Raises:
+            ConfigOptionsLoadError if there are any errors during the parsing of the TOML file
+        """
+        loaded = toml.load(filename)
+        name = self._name
+        if name not in loaded:
+            raise ConfigOptionsLoadError(f"[{name}] section missing from {filename}")
+
+        for attr in loaded[name]:
+            if attr not in self._attrs:
+                raise ConfigOptionsLoadError(
+                    f"Unknown option: {attr} = {loaded[name][attr]}"
+                )
+            val = loaded[name][attr]
+            if not override:
+                # use value from self if set
+                val = getattr(self, attr) or val
+            if type(self._attrs[attr]) == tuple:
+                val = tuple(val)
+            setattr(self, attr, val)
+        return self
+
+    def asdict(self):
+        return {attr: getattr(self, attr) for attr in sorted(self._attrs.keys())}

osxphotos/datetime_formatter.py

@@ -12,41 +12,59 @@ class DateTimeFormatter:
     @property
     def date(self):
         """ ISO date in form 2020-03-22 """
-        date = self.dt.date().isoformat()
-        return date
+        return self.dt.date().isoformat()
 
     @property
     def year(self):
         """ 4 digit year """
-        year = f"{self.dt.year}"
-        return year
+        return f"{self.dt.year}"
 
     @property
     def yy(self):
         """ 2 digit year """
-        yy = f"{self.dt.strftime('%y')}"
-        return yy
+        return f"{self.dt.strftime('%y')}"
 
     @property
     def mm(self):
         """ 2 digit month """
-        mm = f"{self.dt.strftime('%m')}"
-        return mm
+        return f"{self.dt.strftime('%m')}"
 
     @property
     def month(self):
         """ Month as locale's full name """
-        month = f"{self.dt.strftime('%B')}"
-        return month
+        return f"{self.dt.strftime('%B')}"
 
     @property
     def mon(self):
         """ Month as locale's abbreviated name """
-        mon = f"{self.dt.strftime('%b')}"
-        return mon
+        return f"{self.dt.strftime('%b')}"
+
+    @property
+    def dd(self):
+        """ 2-digit day of the month """
+        return f"{self.dt.strftime('%d')}"
+
+    @property
+    def dow(self):
+        """ Day of week as locale's name """
+        return f"{self.dt.strftime('%A')}"
 
     @property
     def doy(self):
         """ Julian day of year starting from 001 """
-        doy = f"{self.dt.strftime('%j')}"
-        return doy
+        return f"{self.dt.strftime('%j')}"
+
+    @property
+    def hour(self):
+        """ 2-digit hour """
+        return f"{self.dt.strftime('%H')}"
+
+    @property
+    def min(self):
+        """ 2-digit minute """
+        return f"{self.dt.strftime('%M')}"
+
+    @property
+    def sec(self):
+        """ 2-digit second """
+        return f"{self.dt.strftime('%S')}"

osxphotos/datetime_utils.py

@@ -0,0 +1,161 @@
+""" datetime.datetime helper functions for converting to/from UTC """
+
+import datetime
+
+
+def get_local_tz(dt):
+    """ Return local timezone as datetime.timezone tzinfo for dt
+    
+    Args:
+        dt: datetime.datetime
+    
+    Returns:
+        local timezone for dt as datetime.timezone
+
+    Raises:
+        ValueError if dt is not timezone naive
+    """
+    if not datetime_has_tz(dt):
+        return dt.astimezone().tzinfo
+    else:
+        raise ValueError("dt must be naive datetime.datetime object")
+
+
+def datetime_has_tz(dt):
+    """ Return True if datetime dt has tzinfo else False
+
+    Args:
+        dt: datetime.datetime
+    
+    Returns:
+        True if dt is timezone aware, else False
+        
+    Raises:
+        TypeError if dt is not a datetime.datetime object
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None
+
+
+def datetime_tz_to_utc(dt):
+    """ Convert datetime.datetime object with timezone to UTC timezone 
+
+    Args:
+        dt: datetime.datetime object
+
+    Returns:
+        datetime.datetime in UTC timezone
+        
+    Raises: 
+        TypeError if dt is not datetime.datetime object
+        ValueError if dt does not have timeone information
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None:
+        return dt.replace(tzinfo=dt.tzinfo).astimezone(tz=datetime.timezone.utc)
+    else:
+        raise ValueError(f"dt does not have timezone info")
+
+
+def datetime_remove_tz(dt):
+    """ Remove timezone from a datetime.datetime object
+
+    Args:
+        dt: datetime.datetime object with tzinfo
+    
+    Returns:
+        dt without any timezone info (naive datetime object) 
+        
+    Raises:
+        TypeError if dt is not a datetime.datetime object
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    return dt.replace(tzinfo=None)
+
+
+def datetime_naive_to_utc(dt):
+    """ Convert naive (timezone unaware) datetime.datetime
+        to aware timezone in UTC timezone
+
+    Args:
+        dt: datetime.datetime without timezone
+    
+    Returns:
+        datetime.datetime with UTC timezone
+        
+    Raises:
+        TypeError if dt is not a datetime.datetime object
+        ValueError if dt is not a naive/timezone unaware object
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None:
+        # has timezone info
+        raise ValueError(
+            "dt must be naive/timezone unaware: "
+            f"{dt} has tzinfo {dt.tzinfo} and offset {dt.tzinfo.utcoffset(dt)}"
+        )
+
+    return dt.replace(tzinfo=datetime.timezone.utc)
+
+
+def datetime_naive_to_local(dt):
+    """ Convert naive (timezone unaware) datetime.datetime
+        to aware timezone in local timezone
+
+    Args:
+        dt: datetime.datetime without timezone
+    
+    Returns:
+        datetime.datetime with local timezone
+        
+    Raises:
+        TypeError if dt is not a datetime.datetime object
+        ValueError if dt is not a naive/timezone unaware object
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None:
+        # has timezone info
+        raise ValueError(
+            "dt must be naive/timezone unaware: "
+            f"{dt} has tzinfo {dt.tzinfo} and offset {dt.tizinfo.utcoffset(dt)}"
+        )
+
+    return dt.replace(tzinfo=get_local_tz(dt))
+
+
+def datetime_utc_to_local(dt):
+    """ Convert datetime.datetime object in UTC timezone to local timezone 
+
+    Args:
+        dt: datetime.datetime object
+
+    Returns:
+        datetime.datetime in local timezone
+
+    Raises:
+        TypeError if dt is not a datetime.datetime object
+        ValueError if dt is not in UTC timezone
+    """
+
+    if type(dt) != datetime.datetime:
+        raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}")
+
+    if dt.tzinfo is not datetime.timezone.utc:
+        raise ValueError(f"{dt} must be in UTC timezone: timezone = {dt.tzinfo}")
+
+    return dt.replace(tzinfo=datetime.timezone.utc).astimezone(tz=None)

osxphotos/exiftool.py

@@ -2,18 +2,18 @@
     I rolled my own for following reasons: 
     1. I wanted something under MIT license (best alternative was licensed under GPL/BSD)
     2. I wanted singleton behavior so only a single exiftool process was ever running
+    3. When used as a context manager, I wanted the operations to batch until exiting the context (improved performance)
     If these aren't important to you, I highly recommend you use Sven Marnach's excellent 
     pyexiftool: https://github.com/smarnach/pyexiftool which provides more functionality """
 
 import json
 import logging
 import os
+import re
+import shutil
 import subprocess
-import sys
 from functools import lru_cache  # pylint: disable=syntax-error
 
-from .utils import _debug
-
 # exiftool -stay_open commands outputs this EOF marker after command is run
 EXIFTOOL_STAYOPEN_EOF = "{ready}"
 EXIFTOOL_STAYOPEN_EOF_LEN = len(EXIFTOOL_STAYOPEN_EOF)
@@ -22,10 +22,7 @@ EXIFTOOL_STAYOPEN_EOF_LEN = len(EXIFTOOL_STAYOPEN_EOF)
 @lru_cache(maxsize=1)
 def get_exiftool_path():
     """ return path of exiftool, cache result """
-    result = subprocess.run(["which", "exiftool"], stdout=subprocess.PIPE)
-    exiftool_path = result.stdout.decode("utf-8")
-    if _debug():
-        logging.debug("exiftool path = %s" % (exiftool_path))
+    exiftool_path = shutil.which("exiftool")
     if exiftool_path:
         return exiftool_path.rstrip()
     else:
@@ -36,8 +33,8 @@ def get_exiftool_path():
 
 
 class _ExifToolProc:
-    """ Runs exiftool in a subprocess via Popen
-        Creates a singleton object """
+    """Runs exiftool in a subprocess via Popen
+    Creates a singleton object"""
 
     def __new__(cls, *args, **kwargs):
         """ create new object or return instance of already created singleton """
@@ -47,24 +44,20 @@ class _ExifToolProc:
         return cls.instance
 
     def __init__(self, exiftool=None):
-        """ construct _ExifToolProc singleton object or return instance of already created object
-            exiftool: optional path to exiftool binary (if not provided, will search path to find it) """
+        """construct _ExifToolProc singleton object or return instance of already created object
+        exiftool: optional path to exiftool binary (if not provided, will search path to find it)"""
 
         if hasattr(self, "_process_running") and self._process_running:
             # already running
-            if exiftool is not None:
+            if exiftool is not None and exiftool != self._exiftool:
                 logging.warning(
                     f"exiftool subprocess already running, "
                     f"ignoring exiftool={exiftool}"
                 )
             return
 
-        if exiftool:
-            self._exiftool = exiftool
-        else:
-            self._exiftool = get_exiftool_path()
-
         self._process_running = False
+        self._exiftool = exiftool or get_exiftool_path()
         self._start_proc()
 
     @property
@@ -102,18 +95,19 @@ class _ExifToolProc:
                 "-",  # read from stdin
                 "-common_args",  # specifies args common to all commands subsequently run
                 "-n",  # no print conversion (e.g. print tag values in machine readable format)
+                "-P",  # Preserve file modification date/time
                 "-G",  # print group name for each tag
             ],
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
-            stderr=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
         )
         self._process_running = True
 
     def _stop_proc(self):
         """ stop the exiftool process if it's running, otherwise, do nothing """
+
         if not self._process_running:
-            logging.warning("exiftool process is not running")
             return
 
         self._process.stdin.write(b"-stay_open\n")
@@ -138,40 +132,79 @@ class _ExifToolProc:
 class ExifTool:
     """ Basic exiftool interface for reading and writing EXIF tags """
 
-    def __init__(self, filepath, exiftool=None, overwrite=True):
-        """ Return ExifTool object
+    def __init__(self, filepath, exiftool=None, overwrite=True, flags=None):
+        """Create ExifTool object
+
+        Args:
             file: path to image file
             exiftool: path to exiftool, if not specified will look in path
-            overwrite: if True, will overwrite image file without creating backup, default=False """
+            overwrite: if True, will overwrite image file without creating backup, default=False
+            flags: optional list of exiftool flags to prepend to exiftool command when writing metadata (e.g. -m or -F)
+
+        Returns:
+            ExifTool instance
+        """
         self.file = filepath
         self.overwrite = overwrite
+        self.flags = flags or []
         self.data = {}
+        self.warning = None
+        self.error = None
+        # if running as a context manager, self._context_mgr will be True
+        self._context_mgr = False
         self._exiftoolproc = _ExifToolProc(exiftool=exiftool)
         self._process = self._exiftoolproc.process
         self._read_exif()
 
     def setvalue(self, tag, value):
-        """ Set tag to value(s) 
-            if value is None, will delete tag """
+        """Set tag to value(s); if value is None, will delete tag
+
+        Args:
+            tag: str; name of tag to set
+            value: str; value to set tag to
+
+        Returns:
+            True if success otherwise False
+
+            If error generated by exiftool, returns False and sets self.error to error string
+            If warning generated by exiftool, returns True (unless there was also an error) and sets self.warning to warning string
+            If called in context manager, returns True (execution is delayed until exiting context manager)
+        """
 
         if value is None:
             value = ""
-        command = []
-        command.append(f"-{tag}={value}")
-        if self.overwrite:
+        command = [f"-{tag}={value}"]
+        if self.overwrite and not self._context_mgr:
             command.append("-overwrite_original")
-        self.run_commands(*command)
+        if self._context_mgr:
+            self._commands.extend(command)
+            return True
+        else:
+            _, _, error = self.run_commands(*command)
+            return error is None
 
     def addvalues(self, tag, *values):
-        """ Add one or more value(s) to tag
+        """Add one or more value(s) to tag
             If more than one value is passed, each value will be added to the tag
-            Notes: exiftool may add duplicate values for some tags so the caller must ensure
-                   the values being added are not already in the EXIF data
-                   For some tags, such as IPTC:Keywords, this will add a new value to the list of keywords, 
-                   but for others, such as EXIF:ISO, this will literally add a value to the existing value.
-                   It's up to the caller to know what exiftool will do for each tag
-                   If setvalue called before addvalues, exiftool does not appear to add duplicates, 
-                   but if addvalues called without first calling setvalue, exiftool will add duplicate values
+
+        Args:
+            tag: str; tag to set
+            *values: str; one or more values to set
+
+        Returns:
+            True if success otherwise False
+
+            If error generated by exiftool, returns False and sets self.error to error string
+            If warning generated by exiftool, returns True (unless there was also an error) and sets self.warning to warning string
+            If called in context manager, returns True (execution is delayed until exiting context manager)
+
+        Notes: exiftool may add duplicate values for some tags so the caller must ensure
+               the values being added are not already in the EXIF data
+               For some tags, such as IPTC:Keywords, this will add a new value to the list of keywords,
+               but for others, such as EXIF:ISO, this will literally add a value to the existing value.
+               It's up to the caller to know what exiftool will do for each tag
+               If setvalue called before addvalues, exiftool does not appear to add duplicates,
+               but if addvalues called without first calling setvalue, exiftool will add duplicate values
         """
         if not values:
             raise ValueError("Must pass at least one value")
@@ -182,25 +215,51 @@ class ExifTool:
                 raise ValueError("Can't add None value to tag")
             command.append(f"-{tag}+={value}")
 
-        if self.overwrite:
+        if self.overwrite and not self._context_mgr:
             command.append("-overwrite_original")
 
-        if command:
-            self.run_commands(*command)
+        if self._context_mgr:
+            self._commands.extend(command)
+            return True
+        else:
+            _, _, error = self.run_commands(*command)
+            return error is None
 
     def run_commands(self, *commands, no_file=False):
-        """ run commands in the exiftool process and return result 
-            no_file: (bool) do not pass the filename to exiftool (default=False)
-                     by default, all commands will be run against self.file
-                     use no_file=True to run a command without passing the filename """
-        if not hasattr(self, "_process") or not self._process:
+        """Run commands in the exiftool process and return result.
+
+        Args:
+                *commands: exiftool commands to run
+                no_file: (bool) do not pass the filename to exiftool (default=False)
+                        by default, all commands will be run against self.file
+                        use no_file=True to run a command without passing the filename
+        Returns:
+            (output, warning, errror)
+            output: bytes is containing output of exiftool commands
+            warning: if exiftool generated warnings, string containing warning otherwise empty string
+            error: if exiftool generated errors, string containing otherwise empty string
+
+        Note: Also sets self.warning and self.error if warning or error generated.
+        """
+        if not (hasattr(self, "_process") and self._process):
             raise ValueError("exiftool process is not running")
 
         if not commands:
             raise TypeError("must provide one or more command to run")
 
+        if self._context_mgr and self.overwrite:
+            commands = list(commands)
+            commands.append("-overwrite_original")
+
         filename = os.fsencode(self.file) if not no_file else b""
-        command_str = (
+
+        if self.flags:
+            command_str = b"\n".join([f.encode("utf-8") for f in self.flags])
+            command_str += b"\n"
+        else:
+            command_str = b""
+
+        command_str += (
             b"\n".join([c.encode("utf-8") for c in commands])
             + b"\n"
             + filename
@@ -208,18 +267,27 @@ class ExifTool:
             + b"-execute\n"
         )
 
-        if _debug():
-            logging.debug(command_str)
-
         # send the command
         self._process.stdin.write(command_str)
         self._process.stdin.flush()
 
         # read the output
         output = b""
+        warning = b""
+        error = b""
         while EXIFTOOL_STAYOPEN_EOF not in str(output):
-            output += self._process.stdout.readline().strip()
-        return output[:-EXIFTOOL_STAYOPEN_EOF_LEN]
+            line = self._process.stdout.readline()
+            if line.startswith(b"Warning"):
+                warning += line.strip()
+            elif line.startswith(b"Error"):
+                error += line.strip()
+            else:
+                output += line.strip()
+        warning = "" if warning == b"" else warning.decode("utf-8")
+        error = "" if error == b"" else error.decode("utf-8")
+        self.warning = warning
+        self.error = error
+        return output[:-EXIFTOOL_STAYOPEN_EOF_LEN], warning, error
 
     @property
     def pid(self):
@@ -229,30 +297,52 @@ class ExifTool:
     @property
     def version(self):
         """ returns exiftool version """
-        ver = self.run_commands("-ver", no_file=True)
+        ver, _, _ = self.run_commands("-ver", no_file=True)
         return ver.decode("utf-8")
 
-    def as_dict(self):
-        """ return dictionary of all EXIF tags and values from exiftool 
-            returns empty dict if no tags
+    def asdict(self, tag_groups=True):
+        """return dictionary of all EXIF tags and values from exiftool
+        returns empty dict if no tags
+
+        Args:
+            tag_groups: if True (default), dict keys have tag groups, e.g. "IPTC:Keywords"; if False, drops groups from keys, e.g. "Keywords"
         """
-        json_str = self.run_commands("-json")
-        if json_str:
-            exifdict = json.loads(json_str)
-            return exifdict[0]
-        else:
+        json_str, _, _ = self.run_commands("-json")
+        if not json_str:
             return dict()
 
+        exifdict = json.loads(json_str)
+        exifdict = exifdict[0]
+        if not tag_groups:
+            # strip tag groups
+            exif_new = {}
+            for k, v in exifdict.items():
+                k = re.sub(r".*:", "", k)
+                exif_new[k] = v
+            exifdict = exif_new
+        return exifdict
+
     def json(self):
         """ returns JSON string containing all EXIF tags and values from exiftool """
-        json_str = self.run_commands("-json")
-        return json_str
+        json, _, _ = self.run_commands("-json")
+        return json
 
     def _read_exif(self):
         """ read exif data from file """
-        data = self.as_dict()
+        data = self.asdict()
         self.data = {k: v for k, v in data.items()}
 
     def __str__(self):
-        str_ = f"file: {self.file}\nexiftool: {self._exiftoolproc._exiftool}"
-        return str_
+        return f"file: {self.file}\nexiftool: {self._exiftoolproc._exiftool}"
+
+    def __enter__(self):
+        self._context_mgr = True
+        self._commands = []
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type:
+            return False
+        elif self._commands:
+            # run_commands sets self.warning and self.error as needed
+            self.run_commands(*self._commands)

osxphotos/export_db.py

@@ -0,0 +1,744 @@
+""" Helper class for managing a database used by 
+    PhotoInfo.export for tracking state of exports and updates
+"""
+
+import datetime
+import logging
+import os
+import pathlib
+import sqlite3
+import sys
+from abc import ABC, abstractmethod
+from io import StringIO
+from sqlite3 import Error
+
+from ._version import __version__
+
+OSXPHOTOS_EXPORTDB_VERSION = "3.2"
+
+
+class ExportDB_ABC(ABC):
+    """ abstract base class for ExportDB """
+
+    @abstractmethod
+    def get_uuid_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def set_uuid_for_file(self, filename, uuid):
+        pass
+
+    @abstractmethod
+    def set_stat_orig_for_file(self, filename, stats):
+        pass
+
+    @abstractmethod
+    def get_stat_orig_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def set_stat_edited_for_file(self, filename, stats):
+        pass
+
+    @abstractmethod
+    def get_stat_edited_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def set_stat_converted_for_file(self, filename, stats):
+        pass
+
+    @abstractmethod
+    def get_stat_converted_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def set_stat_exif_for_file(self, filename, stats):
+        pass
+
+    @abstractmethod
+    def get_stat_exif_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def get_info_for_uuid(self, uuid):
+        pass
+
+    @abstractmethod
+    def set_info_for_uuid(self, uuid, info):
+        pass
+
+    @abstractmethod
+    def get_exifdata_for_file(self, uuid):
+        pass
+
+    @abstractmethod
+    def set_exifdata_for_file(self, uuid, exifdata):
+        pass
+
+    @abstractmethod
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        pass
+
+    @abstractmethod
+    def get_sidecar_for_file(self, filename):
+        pass
+
+    @abstractmethod
+    def set_data(
+        self,
+        filename,
+        uuid,
+        orig_stat,
+        exif_stat,
+        converted_stat,
+        edited_stat,
+        info_json,
+        exif_json,
+    ):
+        pass
+
+
+class ExportDBNoOp(ExportDB_ABC):
+    """ An ExportDB with NoOp methods """
+
+    def __init__(self):
+        self.was_created = True
+        self.was_upgraded = False
+        self.version = OSXPHOTOS_EXPORTDB_VERSION
+
+    def get_uuid_for_file(self, filename):
+        pass
+
+    def set_uuid_for_file(self, filename, uuid):
+        pass
+
+    def set_stat_orig_for_file(self, filename, stats):
+        pass
+
+    def get_stat_orig_for_file(self, filename):
+        pass
+
+    def set_stat_edited_for_file(self, filename, stats):
+        pass
+
+    def get_stat_edited_for_file(self, filename):
+        pass
+
+    def set_stat_converted_for_file(self, filename, stats):
+        pass
+
+    def get_stat_converted_for_file(self, filename):
+        pass
+
+    def set_stat_exif_for_file(self, filename, stats):
+        pass
+
+    def get_stat_exif_for_file(self, filename):
+        pass
+
+    def get_info_for_uuid(self, uuid):
+        pass
+
+    def set_info_for_uuid(self, uuid, info):
+        pass
+
+    def get_exifdata_for_file(self, uuid):
+        pass
+
+    def set_exifdata_for_file(self, uuid, exifdata):
+        pass
+
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        pass
+
+    def get_sidecar_for_file(self, filename):
+        return None, (None, None, None)
+
+    def set_data(
+        self,
+        filename,
+        uuid,
+        orig_stat,
+        exif_stat,
+        converted_stat,
+        edited_stat,
+        info_json,
+        exif_json,
+    ):
+        pass
+
+
+class ExportDB(ExportDB_ABC):
+    """ Interface to sqlite3 database used to store state information for osxphotos export command """
+
+    def __init__(self, dbfile):
+        """ dbfile: path to osxphotos export database file """
+        self._dbfile = dbfile
+        # _path is parent of the database
+        # all files referenced by get_/set_uuid_for_file will be converted to
+        # relative paths to this parent _path
+        # this allows the entire export tree to be moved to a new disk/location
+        # whilst preserving the UUID to filename mappping
+        self._path = pathlib.Path(dbfile).parent
+        self._conn = self._open_export_db(dbfile)
+        self._insert_run_info()
+
+    def get_uuid_for_file(self, filename):
+        """ query database for filename and return UUID
+            returns None if filename not found in database
+        """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                f"SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,)
+            )
+            results = c.fetchone()
+            uuid = results[0] if results else None
+        except Error as e:
+            logging.warning(e)
+            uuid = None
+
+        return uuid
+
+    def set_uuid_for_file(self, filename, uuid):
+        """ set UUID of filename to uuid in the database """
+        filename = str(pathlib.Path(filename).relative_to(self._path))
+        filename_normalized = filename.lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);",
+                (filename, filename_normalized, uuid),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def set_stat_orig_for_file(self, filename, stats):
+        """ set stat info for filename
+            filename: filename to set the stat info for
+            stat: a tuple of length 3: mode, size, mtime """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        if len(stats) != 3:
+            raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
+
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "UPDATE files "
+                + "SET orig_mode = ?, orig_size = ?, orig_mtime = ? "
+                + "WHERE filepath_normalized = ?;",
+                (*stats, filename),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def get_stat_orig_for_file(self, filename):
+        """ get stat info for filename
+            returns: tuple of (mode, size, mtime) 
+        """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT orig_mode, orig_size, orig_mtime FROM files WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            if results:
+                stats = results[0:3]
+                mtime = int(stats[2]) if stats[2] is not None else None
+                stats = (stats[0], stats[1], mtime)
+            else:
+                stats = (None, None, None)
+        except Error as e:
+            logging.warning(e)
+            stats = (None, None, None)
+
+        return stats
+
+    def set_stat_edited_for_file(self, filename, stats):
+        """ set stat info for edited version of image (in Photos' library)
+            filename: filename to set the stat info for
+            stat: a tuple of length 3: mode, size, mtime """
+        return self._set_stat_for_file("edited", filename, stats)
+
+    def get_stat_edited_for_file(self, filename):
+        """ get stat info for edited version of image (in Photos' library)
+            filename: filename to set the stat info for
+            stat: a tuple of length 3: mode, size, mtime """
+        return self._get_stat_for_file("edited", filename)
+
+    def set_stat_exif_for_file(self, filename, stats):
+        """ set stat info for filename (after exiftool has updated it)
+            filename: filename to set the stat info for
+            stat: a tuple of length 3: mode, size, mtime """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        if len(stats) != 3:
+            raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
+
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "UPDATE files "
+                + "SET exif_mode = ?, exif_size = ?, exif_mtime = ? "
+                + "WHERE filepath_normalized = ?;",
+                (*stats, filename),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def get_stat_exif_for_file(self, filename):
+        """ get stat info for filename (after exiftool has updated it)
+            returns: tuple of (mode, size, mtime) 
+        """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT exif_mode, exif_size, exif_mtime FROM files WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            if results:
+                stats = results[0:3]
+                mtime = int(stats[2]) if stats[2] is not None else None
+                stats = (stats[0], stats[1], mtime)
+            else:
+                stats = (None, None, None)
+        except Error as e:
+            logging.warning(e)
+            stats = (None, None, None)
+
+        return stats
+
+    def set_stat_converted_for_file(self, filename, stats):
+        """ set stat info for filename (after image converted to jpeg)
+            filename: filename to set the stat info for
+            stat: a tuple of length 3: mode, size, mtime """
+        return self._set_stat_for_file("converted", filename, stats)
+
+    def get_stat_converted_for_file(self, filename):
+        """ get stat info for filename (after jpeg conversion)
+            returns: tuple of (mode, size, mtime) 
+        """
+        return self._get_stat_for_file("converted", filename)
+
+    def get_info_for_uuid(self, uuid):
+        """ returns the info JSON struct for a UUID """
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute("SELECT json_info FROM info WHERE uuid = ?", (uuid,))
+            results = c.fetchone()
+            info = results[0] if results else None
+        except Error as e:
+            logging.warning(e)
+            info = None
+
+        return info
+
+    def set_info_for_uuid(self, uuid, info):
+        """ sets the info JSON struct for a UUID """
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "INSERT OR REPLACE INTO info(uuid, json_info) VALUES (?, ?);",
+                (uuid, info),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def get_exifdata_for_file(self, filename):
+        """ returns the exifdata JSON struct for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT json_exifdata FROM exifdata WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            exifdata = results[0] if results else None
+        except Error as e:
+            logging.warning(e)
+            exifdata = None
+
+        return exifdata
+
+    def set_exifdata_for_file(self, filename, exifdata):
+        """ sets the exifdata JSON struct for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
+                (filename, exifdata),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def get_sidecar_for_file(self, filename):
+        """ returns the sidecar data and signature for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT sidecar_data, mode, size, mtime FROM sidecar WHERE filepath_normalized = ?",
+                (filename,),
+            )
+            results = c.fetchone()
+            if results:
+                sidecar_data = results[0]
+                sidecar_sig = (
+                    results[1],
+                    results[2],
+                    int(results[3]) if results[3] is not None else None,
+                )
+            else:
+                sidecar_data = None
+                sidecar_sig = (None, None, None)
+        except Error as e:
+            logging.warning(e)
+            sidecar_data = None
+            sidecar_sig = (None, None, None)
+
+        return sidecar_data, sidecar_sig
+
+    def set_sidecar_for_file(self, filename, sidecar_data, sidecar_sig):
+        """ sets the sidecar data and signature for a file """
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                "INSERT OR REPLACE INTO sidecar(filepath_normalized, sidecar_data, mode, size, mtime) VALUES (?, ?, ?, ?, ?);",
+                (filename, sidecar_data, *sidecar_sig),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def set_data(
+        self,
+        filename,
+        uuid,
+        orig_stat,
+        exif_stat,
+        converted_stat,
+        edited_stat,
+        info_json,
+        exif_json,
+    ):
+        """ sets all the data for file and uuid at once 
+        """
+        filename = str(pathlib.Path(filename).relative_to(self._path))
+        filename_normalized = filename.lower()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);",
+                (filename, filename_normalized, uuid),
+            )
+            c.execute(
+                "UPDATE files "
+                + "SET orig_mode = ?, orig_size = ?, orig_mtime = ? "
+                + "WHERE filepath_normalized = ?;",
+                (*orig_stat, filename_normalized),
+            )
+            c.execute(
+                "UPDATE files "
+                + "SET exif_mode = ?, exif_size = ?, exif_mtime = ? "
+                + "WHERE filepath_normalized = ?;",
+                (*exif_stat, filename_normalized),
+            )
+            c.execute(
+                "INSERT OR REPLACE INTO converted(filepath_normalized, mode, size, mtime) VALUES (?, ?, ?, ?);",
+                (filename_normalized, *converted_stat),
+            )
+            c.execute(
+                "INSERT OR REPLACE INTO edited(filepath_normalized, mode, size, mtime) VALUES (?, ?, ?, ?);",
+                (filename_normalized, *edited_stat),
+            )
+            c.execute(
+                "INSERT OR REPLACE INTO info(uuid, json_info) VALUES (?, ?);",
+                (uuid, info_json),
+            )
+            c.execute(
+                "INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
+                (filename_normalized, exif_json),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def close(self):
+        """ close the database connection """
+        try:
+            self._conn.close()
+        except Error as e:
+            logging.warning(e)
+
+    def _set_stat_for_file(self, table, filename, stats):
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        if len(stats) != 3:
+            raise ValueError(f"expected 3 elements for stat, got {len(stats)}")
+
+        conn = self._conn
+        c = conn.cursor()
+        c.execute(
+            f"INSERT OR REPLACE INTO {table}(filepath_normalized, mode, size, mtime) VALUES (?, ?, ?, ?);",
+            (filename, *stats),
+        )
+        conn.commit()
+
+    def _get_stat_for_file(self, table, filename):
+        filename = str(pathlib.Path(filename).relative_to(self._path)).lower()
+        conn = self._conn
+        c = conn.cursor()
+        c.execute(
+            f"SELECT mode, size, mtime FROM {table} WHERE filepath_normalized = ?",
+            (filename,),
+        )
+        results = c.fetchone()
+        if results:
+            stats = results[0:3]
+            mtime = int(stats[2]) if stats[2] is not None else None
+            stats = (stats[0], stats[1], mtime)
+        else:
+            stats = (None, None, None)
+
+        return stats
+
+    def _open_export_db(self, dbfile):
+        """ open export database and return a db connection
+            if dbfile does not exist, will create and initialize the database 
+            returns: connection to the database 
+        """
+
+        if not os.path.isfile(dbfile):
+            conn = self._get_db_connection(dbfile)
+            if not conn:
+                raise Exception("Error getting connection to database {dbfile}")
+            self._create_db_tables(conn)
+            self.was_created = True
+            self.was_upgraded = ()
+        else:
+            conn = self._get_db_connection(dbfile)
+            self.was_created = False
+            version_info = self._get_database_version(conn)
+            if version_info[1] < OSXPHOTOS_EXPORTDB_VERSION:
+                self._create_db_tables(conn)
+                self.was_upgraded = (version_info[1], OSXPHOTOS_EXPORTDB_VERSION)
+            else:
+                self.was_upgraded = ()
+        self.version = OSXPHOTOS_EXPORTDB_VERSION
+        return conn
+
+    def _get_db_connection(self, dbfile):
+        """ return db connection to dbname """
+        try:
+            conn = sqlite3.connect(dbfile)
+        except Error as e:
+            logging.warning(e)
+            conn = None
+
+        return conn
+
+    def _get_database_version(self, conn):
+        """ return tuple of (osxphotos, exportdb) versions for database connection conn """
+        version_info = conn.execute(
+            "SELECT osxphotos, exportdb, max(id) FROM version"
+        ).fetchone()
+        return (version_info[0], version_info[1])
+
+    def _create_db_tables(self, conn):
+        """ create (if not already created) the necessary db tables for the export database
+            conn: sqlite3 db connection 
+        """
+        sql_commands = {
+            "sql_version_table": """ CREATE TABLE IF NOT EXISTS version (
+                                id INTEGER PRIMARY KEY,
+                                osxphotos TEXT,
+                                exportdb TEXT 
+                                ); """,
+            "sql_files_table": """ CREATE TABLE IF NOT EXISTS files (
+                              id INTEGER PRIMARY KEY,
+                              filepath TEXT NOT NULL,
+                              filepath_normalized TEXT NOT NULL,
+                              uuid TEXT,
+                              orig_mode INTEGER,
+                              orig_size INTEGER,
+                              orig_mtime REAL,
+                              exif_mode INTEGER,
+                              exif_size INTEGER,
+                              exif_mtime REAL
+                              ); """,
+            "sql_runs_table": """ CREATE TABLE IF NOT EXISTS runs (
+                             id INTEGER PRIMARY KEY,
+                             datetime TEXT,
+                             python_path TEXT,
+                             script_name TEXT,
+                             args TEXT,
+                             cwd TEXT 
+                             ); """,
+            "sql_info_table": """ CREATE TABLE IF NOT EXISTS info (
+                             id INTEGER PRIMARY KEY,
+                             uuid text NOT NULL,
+                             json_info JSON 
+                             ); """,
+            "sql_exifdata_table": """ CREATE TABLE IF NOT EXISTS exifdata (
+                             id INTEGER PRIMARY KEY,
+                             filepath_normalized TEXT NOT NULL,
+                             json_exifdata JSON 
+                             ); """,
+            "sql_edited_table": """ CREATE TABLE IF NOT EXISTS edited (
+                              id INTEGER PRIMARY KEY,
+                              filepath_normalized TEXT NOT NULL,
+                              mode INTEGER,
+                              size INTEGER,
+                              mtime REAL
+                              ); """,
+            "sql_converted_table": """ CREATE TABLE IF NOT EXISTS converted (
+                              id INTEGER PRIMARY KEY,
+                              filepath_normalized TEXT NOT NULL,
+                              mode INTEGER,
+                              size INTEGER,
+                              mtime REAL
+                              ); """,
+            "sql_sidecar_table": """ CREATE TABLE IF NOT EXISTS sidecar (
+                              id INTEGER PRIMARY KEY,
+                              filepath_normalized TEXT NOT NULL,
+                              sidecar_data TEXT,
+                              mode INTEGER,
+                              size INTEGER,
+                              mtime REAL
+                              ); """,
+            "sql_files_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_files_filepath_normalized on files (filepath_normalized); """,
+            "sql_info_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_info_uuid on info (uuid); """,
+            "sql_exifdata_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_exifdata_filename on exifdata (filepath_normalized); """,
+            "sql_edited_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_edited_filename on edited (filepath_normalized);""",
+            "sql_converted_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_converted_filename on converted (filepath_normalized);""",
+            "sql_sidecar_idx": """ CREATE UNIQUE INDEX IF NOT EXISTS idx_sidecar_filename on sidecar (filepath_normalized);""",
+        }
+        try:
+            c = conn.cursor()
+            for cmd in sql_commands.values():
+                c.execute(cmd)
+            c.execute(
+                "INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);",
+                (__version__, OSXPHOTOS_EXPORTDB_VERSION),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+    def __del__(self):
+        """ ensure the database connection is closed """
+        try:
+            self._conn.close()
+        except:
+            pass
+
+    def _insert_run_info(self):
+        dt = datetime.datetime.utcnow().isoformat()
+        python_path = sys.executable
+        cmd = sys.argv[0]
+        args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else ""
+        cwd = os.getcwd()
+        conn = self._conn
+        try:
+            c = conn.cursor()
+            c.execute(
+                f"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)",
+                (dt, python_path, cmd, args, cwd),
+            )
+            conn.commit()
+        except Error as e:
+            logging.warning(e)
+
+
+class ExportDBInMemory(ExportDB):
+    """ In memory version of ExportDB
+        Copies the on-disk database into memory so it may be operated on without 
+        modifying the on-disk verison 
+    """
+
+    def init(self, dbfile):
+        self._dbfile = dbfile
+        # _path is parent of the database
+        # all files referenced by get_/set_uuid_for_file will be converted to
+        # relative paths to this parent _path
+        # this allows the entire export tree to be moved to a new disk/location
+        # whilst preserving the UUID to filename mappping
+        self._path = pathlib.Path(dbfile).parent
+        self._conn = self._open_export_db(dbfile)
+        self._insert_run_info()
+
+    def _open_export_db(self, dbfile):
+        """ open export database and return a db connection
+            returns: connection to the database 
+        """
+        if not os.path.isfile(dbfile):
+            conn = self._get_db_connection()
+            if conn:
+                self._create_db_tables(conn)
+                self.was_created = True
+                self.was_upgraded = ()
+                self.version = OSXPHOTOS_EXPORTDB_VERSION
+            else:
+                raise Exception("Error getting connection to in-memory database")
+        else:
+            try:
+                conn = sqlite3.connect(dbfile)
+            except Error as e:
+                logging.warning(e)
+                raise e
+
+            tempfile = StringIO()
+            for line in conn.iterdump():
+                tempfile.write("%s\n" % line)
+            conn.close()
+            tempfile.seek(0)
+
+            # Create a database in memory and import from tempfile
+            conn = sqlite3.connect(":memory:")
+            conn.cursor().executescript(tempfile.read())
+            conn.commit()
+            self.was_created = False
+            _, exportdb_ver = self._get_database_version(conn)
+            if exportdb_ver < OSXPHOTOS_EXPORTDB_VERSION:
+                self._create_db_tables(conn)
+                self.was_upgraded = (exportdb_ver, OSXPHOTOS_EXPORTDB_VERSION)
+            else:
+                self.was_upgraded = ()
+            self.version = OSXPHOTOS_EXPORTDB_VERSION
+
+        return conn
+
+    def _get_db_connection(self):
+        """ return db connection to in memory database """
+        try:
+            conn = sqlite3.connect(":memory:")
+        except Error as e:
+            logging.warning(e)
+            conn = None
+
+        return conn

osxphotos/fileutil.py

@@ -0,0 +1,292 @@
+""" FileUtil class with methods for copy, hardlink, unlink, etc. """
+
+import os
+import pathlib
+import stat
+import subprocess
+import sys
+from abc import ABC, abstractmethod
+
+import CoreFoundation
+
+from .imageconverter import ImageConverter
+
+
+class FileUtilABC(ABC):
+    """ Abstract base class for FileUtil """
+
+    @classmethod
+    @abstractmethod
+    def hardlink(cls, src, dest):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def copy(cls, src, dest, norsrc=False):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def unlink(cls, dest):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def rmdir(cls, dest):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def utime(cls, path, times):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def cmp(cls, file1, file2, mtime1=None):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def cmp_file_sig(cls, file1, file2):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def file_sig(cls, file1):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def convert_to_jpeg(cls, src_file, dest_file, compression_quality=1.0):
+        pass
+
+    @classmethod
+    @abstractmethod
+    def rename(cls, src, dest):
+        pass
+
+
+class FileUtilMacOS(FileUtilABC):
+    """ Various file utilities """
+
+    @classmethod
+    def hardlink(cls, src, dest):
+        """ Hardlinks a file from src path to dest path 
+            src: source path as string 
+            dest: destination path as string
+            Raises exception if linking fails or either path is None """
+
+        if src is None or dest is None:
+            raise ValueError("src and dest must not be None", src, dest)
+
+        if not os.path.isfile(src):
+            raise FileNotFoundError("src file does not appear to exist", src)
+
+        # if error on copy, subprocess will raise CalledProcessError
+        try:
+            os.link(src, dest)
+        except Exception as e:
+            raise e
+
+    @classmethod
+    def copy(cls, src, dest):
+        """ Copies a file from src path to dest path 
+        
+        Args:
+            src: source path as string; must be a valid file path
+            dest: destination path as string
+                  dest may be either directory or file; in either case, src file must not exist in dest
+            Note: src and dest may be either a string or a pathlib.Path object
+        
+        Returns:
+            True if copy succeeded
+        
+        Raises:
+            OSError if copy fails
+            TypeError if either path is None
+        """
+        if not isinstance(src, pathlib.Path):
+            src = pathlib.Path(src)
+
+        if not isinstance(dest, pathlib.Path):
+            dest = pathlib.Path(dest)
+
+        if dest.is_dir():
+            dest /= src.name
+
+        filemgr = CoreFoundation.NSFileManager.defaultManager()
+        error = filemgr.copyItemAtPath_toPath_error_(str(src), str(dest), None)
+        # error is a tuple of (bool, error_string)
+        # error[0] is True if copy succeeded
+        if not error[0]:
+            raise OSError(error[1])
+        return True
+
+    @classmethod
+    def unlink(cls, filepath):
+        """ unlink filepath; if it's pathlib.Path, use Path.unlink, otherwise use os.unlink """
+        if isinstance(filepath, pathlib.Path):
+            filepath.unlink()
+        else:
+            os.unlink(filepath)
+
+    @classmethod
+    def rmdir(cls, dirpath):
+        """ remove directory filepath; dirpath must be empty """
+        if isinstance(dirpath, pathlib.Path):
+            dirpath.rmdir()
+        else:
+            os.rmdir(dirpath)
+
+    @classmethod
+    def utime(cls, path, times):
+        """ Set the access and modified time of path. """
+        os.utime(path, times)
+
+    @classmethod
+    def cmp(cls, f1, f2, mtime1=None):
+        """Does shallow compare (file signatures) of f1 to file f2.
+        Arguments:
+        f1 --  File name
+        f2 -- File name
+        mtime1 -- optional, pass alternate file modification timestamp for f1; will be converted to int
+
+        Return value:
+        True if the file signatures as returned by stat are the same, False otherwise. 
+        Does not do a byte-by-byte comparison.
+        """
+
+        s1 = cls._sig(os.stat(f1))
+        if mtime1 is not None:
+            s1 = (s1[0], s1[1], int(mtime1))
+        s2 = cls._sig(os.stat(f2))
+        if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
+            return False
+        return s1 == s2
+
+    @classmethod
+    def cmp_file_sig(cls, f1, s2):
+        """Compare file f1 to signature s2.
+        Arguments:
+        f1 --  File name
+        s2 -- stats as returned by _sig
+
+        Return value:
+        True if the files are the same, False otherwise.
+        """
+
+        if not s2:
+            return False
+
+        s1 = cls._sig(os.stat(f1))
+
+        if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
+            return False
+        return s1 == s2
+
+    @classmethod
+    def file_sig(cls, f1):
+        """ return os.stat signature for file f1 """
+        return cls._sig(os.stat(f1))
+
+    @classmethod
+    def convert_to_jpeg(cls, src_file, dest_file, compression_quality=1.0):
+        """ converts image file src_file to jpeg format as dest_file
+
+            Args:
+                src_file: image file to convert
+                dest_file: destination path to write converted file to
+                compression quality: JPEG compression quality in range 0.0 <= compression_quality <= 1.0; default 1.0 (best quality)
+                
+            Returns:
+                True if success, otherwise False 
+        """
+        converter = ImageConverter()
+        return converter.write_jpeg(
+            src_file, dest_file, compression_quality=compression_quality
+        )
+
+    @classmethod
+    def rename(cls, src, dest):
+        """ Copy src to dest 
+
+        Args:
+            src: path to source file
+            dest: path to destination file
+        
+        Returns:
+            Name of renamed file (dest)
+        
+        """
+        os.rename(str(src), str(dest))
+        return dest
+
+    @staticmethod
+    def _sig(st):
+        """ return tuple of (mode, size, mtime) of file based on os.stat
+            Args:
+                st: os.stat signature
+        """
+        # use int(st.st_mtime) because ditto does not copy fractional portion of mtime
+        return (stat.S_IFMT(st.st_mode), st.st_size, int(st.st_mtime))
+
+
+class FileUtil(FileUtilMacOS):
+    """ Various file utilities """
+
+    pass
+
+
+class FileUtilNoOp(FileUtil):
+    """ No-Op implementation of FileUtil for testing / dry-run mode
+        all methods with exception of cmp, cmp_file_sig and file_cmp are no-op
+        cmp and cmp_file_sig functions as FileUtil methods do
+        file_cmp returns mock data
+    """
+
+    @staticmethod
+    def noop(*args):
+        pass
+
+    verbose = noop
+
+    def __new__(cls, verbose=None):
+        if verbose:
+            if callable(verbose):
+                cls.verbose = verbose
+            else:
+                raise ValueError(f"verbose {verbose} not callable")
+        return super(FileUtilNoOp, cls).__new__(cls)
+
+    @classmethod
+    def hardlink(cls, src, dest):
+        cls.verbose(f"hardlink: {src} {dest}")
+
+    @classmethod
+    def copy(cls, src, dest, norsrc=False):
+        cls.verbose(f"copy: {src} {dest}")
+
+    @classmethod
+    def unlink(cls, dest):
+        cls.verbose(f"unlink: {dest}")
+
+    @classmethod
+    def rmdir(cls, dest):
+        cls.verbose(f"rmdir: {dest}")
+
+    @classmethod
+    def utime(cls, path, times):
+        cls.verbose(f"utime: {path}, {times}")
+
+    @classmethod
+    def file_sig(cls, file1):
+        cls.verbose(f"file_sig: {file1}")
+        return (42, 42, 42)
+
+    @classmethod
+    def convert_to_jpeg(cls, src_file, dest_file, compression_quality=1.0):
+        cls.verbose(f"convert_to_jpeg: {src_file}, {dest_file}, {compression_quality}")
+
+    @classmethod
+    def rename(cls, src, dest):
+        cls.verbose(f"rename: {src}, {dest}")

osxphotos/imageconverter.py

@@ -0,0 +1,126 @@
+""" ImageConverter class
+    Convert an image to JPEG using CoreImage --
+    for example, RAW to JPEG.  Only works if Mac equipped with GPU. """
+
+# reference: https://stackoverflow.com/questions/59330149/coreimage-ciimage-write-jpg-is-shifting-colors-macos/59334308#59334308
+
+import pathlib
+
+import objc
+import Metal
+import Quartz
+from Cocoa import NSURL
+from Foundation import NSDictionary
+
+# needed to capture system-level stderr
+from wurlitzer import pipes
+
+
+class ImageConversionError(Exception):
+    """Base class for exceptions in this module. """
+
+    pass
+
+
+class ImageConverter:
+    """ Convert images to jpeg.  This class is a singleton
+        which will re-use the Core Image CIContext to avoid
+        creating a new context for every conversion. """
+
+    def __new__(cls, *args, **kwargs):
+        """ create new object or return instance of already created singleton """
+        if not hasattr(cls, "instance") or not cls.instance:
+            cls.instance = super().__new__(cls)
+
+        return cls.instance
+
+    def __init__(self):
+        """ return existing singleton or create a new one """
+
+        if hasattr(self, "context"):
+            return
+
+        """ initialize CIContext """
+        context_options = NSDictionary.dictionaryWithDictionary_(
+            {
+                "workingColorSpace": Quartz.CoreGraphics.kCGColorSpaceExtendedSRGB,
+                "workingFormat": Quartz.kCIFormatRGBAh,
+            }
+        )
+        mtldevice = Metal.MTLCreateSystemDefaultDevice()
+        self.context = Quartz.CIContext.contextWithMTLDevice_options_(
+            mtldevice, context_options
+        )
+
+    def write_jpeg(self, input_path, output_path, compression_quality=1.0):
+        """ convert image to jpeg and write image to output_path
+
+        Args:
+            input_path: path to input image (e.g. '/path/to/import/file.CR2') as str or pathlib.Path
+            output_path: path to exported jpeg (e.g. '/path/to/export/file.jpeg') as str or pathlib.Path
+            compression_quality: JPEG compression quality, float in range 0.0 to 1.0; default is 1.0 (best quality)
+
+        Return:
+            True if conversion successful, else False
+
+        Raises:
+            ValueError if compression quality not in range 0.0 to 1.0
+            FileNotFoundError if input_path doesn't exist
+            ImageConversionError if error during conversion
+        """
+
+        # Set up a dedicated objc autorelease pool for this function call.
+        # This is to ensure that all the NSObjects are cleaned up after each
+        # call to prevent memory leaks.
+        # https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmAutoreleasePools.html
+        # https://pyobjc.readthedocs.io/en/latest/api/module-objc.html#memory-management
+        with objc.autorelease_pool():
+            # accept input_path or output_path as pathlib.Path
+            if not isinstance(input_path, str):
+                input_path = str(input_path)
+
+            if not isinstance(output_path, str):
+                output_path = str(output_path)
+
+            if not pathlib.Path(input_path).is_file():
+                raise FileNotFoundError(f"could not find {input_path}")
+
+            if not (0.0 <= compression_quality <= 1.0):
+                raise ValueError(
+                    "illegal value for compression_quality: {compression_quality}"
+                )
+
+            input_url = NSURL.fileURLWithPath_(input_path)
+            output_url = NSURL.fileURLWithPath_(output_path)
+
+            with pipes() as (out, err):
+                # capture stdout and stderr from system calls
+                # otherwise, Quartz.CIImage.imageWithContentsOfURL_
+                # prints to stderr something like:
+                # 2020-09-20 20:55:25.538 python[73042:5650492] Creating client/daemon connection: B8FE995E-3F27-47F4-9FA8-559C615FD774
+                # 2020-09-20 20:55:25.652 python[73042:5650492] Got the query meta data reply for: com.apple.MobileAsset.RawCamera.Camera, response: 0
+                input_image = Quartz.CIImage.imageWithContentsOfURL_(input_url)
+
+            if input_image is None:
+                raise ImageConversionError(f"Could not create CIImage for {input_path}")
+
+            output_colorspace = input_image.colorSpace() or Quartz.CGColorSpaceCreateWithName(
+                Quartz.CoreGraphics.kCGColorSpaceSRGB
+            )
+
+            output_options = NSDictionary.dictionaryWithDictionary_(
+                {"kCGImageDestinationLossyCompressionQuality": compression_quality}
+            )
+            (
+                _,
+                error,
+            ) = self.context.writeJPEGRepresentationOfImage_toURL_colorSpace_options_error_(
+                input_image, output_url, output_colorspace, output_options, None
+            )
+            if not error:
+                return True
+            else:
+                raise ImageConversionError(
+                    "Error converting file {input_path} to jpeg at {output_path}: {error}"
+                )
+

osxphotos/path_utils.py

@@ -0,0 +1,79 @@
+""" utility functions for validating/sanitizing path components """
+
+import pathvalidate
+
+from ._constants import MAX_DIRNAME_LEN, MAX_FILENAME_LEN
+
+
+def sanitize_filepath(filepath):
+    """ sanitize a filepath """
+    return pathvalidate.sanitize_filepath(filepath, platform="macos")
+
+
+def is_valid_filepath(filepath):
+    """ returns True if a filepath is valid otherwise False """
+    return pathvalidate.is_valid_filepath(filepath, platform="macos")
+
+
+def sanitize_filename(filename, replacement=":"):
+    """ replace any illegal characters in a filename and truncate filename if needed
+
+    Args:
+        filename: str, filename to sanitze
+        replacement: str, value to replace any illegal characters with; default = ":"
+    
+    Returns:
+        filename with any illegal characters replaced by replacement and truncated if necessary
+    """
+
+    if filename:
+        filename = filename.replace("/", replacement)
+        if len(filename) > MAX_FILENAME_LEN:
+            parts = filename.split(".")
+            drop = len(filename) - MAX_FILENAME_LEN
+            if len(parts) > 1:
+                # has an extension
+                ext = parts.pop(-1)
+                stem = ".".join(parts)
+                if drop > len(stem):
+                    ext = ext[:-drop]
+                else:
+                    stem = stem[:-drop]
+                filename = f"{stem}.{ext}"
+            else:
+                filename = filename[:-drop]
+    return filename
+
+
+def sanitize_dirname(dirname, replacement=":"):
+    """ replace any illegal characters in a directory name and truncate directory name if needed
+
+    Args:
+        dirname: str, directory name to sanitze
+        replacement: str, value to replace any illegal characters with; default = ":"
+    
+    Returns:
+        dirname with any illegal characters replaced by replacement and truncated if necessary
+    """
+    if dirname:
+        dirname = sanitize_pathpart(dirname, replacement=replacement)
+    return dirname
+
+
+def sanitize_pathpart(pathpart, replacement=":"):
+    """ replace any illegal characters in a path part (either directory or filename without extension) and truncate name if needed
+
+    Args:
+        pathpart: str, path part to sanitze
+        replacement: str, value to replace any illegal characters with; default = ":"
+    
+    Returns:
+        pathpart with any illegal characters replaced by replacement and truncated if necessary
+    """
+    if pathpart:
+        pathpart = pathpart.replace("/", replacement)
+        if len(pathpart) > MAX_DIRNAME_LEN:
+            drop = len(pathpart) - MAX_DIRNAME_LEN
+            pathpart = pathpart[:-drop]
+    return pathpart
+

osxphotos/personinfo.py

@@ -0,0 +1,500 @@
+""" PhotoInfo and FaceInfo classes to expose info about persons and faces in the Photos library """
+
+import json
+import logging
+import math
+
+from collections import namedtuple
+
+MWG_RS_Area = namedtuple("MWG_RS_Area", ["x", "y", "h", "w"])
+MPRI_Reg_Rect = namedtuple("MPRI_Reg_Rect", ["x", "y", "h", "w"])
+
+
+class PersonInfo:
+    """ Info about a person in the Photos library
+    """
+
+    def __init__(self, db=None, pk=None):
+        """ Creates a new PersonInfo instance
+
+        Arguments:
+            db: instance of PhotosDB object
+            pk: primary key value of person to initialize PersonInfo with    
+        
+        Returns:
+            PersonInfo instance
+        """
+        self._db = db
+        self._pk = pk
+
+        person = self._db._dbpersons_pk[pk]
+        self.uuid = person["uuid"]
+        self.name = person["fullname"]
+        self.display_name = person["displayname"]
+        self.keyface = person["keyface"]
+        self.facecount = person["facecount"]
+
+    @property
+    def keyphoto(self):
+        try:
+            return self._keyphoto
+        except AttributeError:
+            person = self._db._dbpersons_pk[self._pk]
+            if person["photo_uuid"]:
+                try:
+                    key_photo = self._db.get_photo(person["photo_uuid"])
+                except IndexError:
+                    key_photo = None
+            else:
+                key_photo = None
+            self._keyphoto = key_photo
+            return self._keyphoto
+
+    @property
+    def photos(self):
+        """ Returns list of PhotoInfo objects associated with this person """
+        return self._db.photos_by_uuid(self._db._dbfaces_pk[self._pk])
+
+    @property
+    def face_info(self):
+        """ Returns a list of FaceInfo objects associated with this person sorted by quality score
+            Highest quality face is result[0] and lowest quality face is result[n] 
+        """
+        try:
+            faces = self._db._db_faceinfo_person[self._pk]
+            return sorted(
+                [FaceInfo(db=self._db, pk=face) for face in faces],
+                key=lambda face: face.quality,
+                reverse=True,
+            )
+        except KeyError:
+            # no faces
+            return []
+
+    def asdict(self):
+        """ Returns dictionary representation of class instance """
+        keyphoto = self.keyphoto.uuid if self.keyphoto is not None else None
+        return {
+            "uuid": self.uuid,
+            "name": self.name,
+            "displayname": self.display_name,
+            "keyface": self.keyface,
+            "facecount": self.facecount,
+            "keyphoto": keyphoto,
+        }
+
+    def json(self):
+        """ Returns JSON representation of class instance """
+        return json.dumps(self.asdict())
+
+    def __str__(self):
+        return f"PersonInfo(name={self.name}, display_name={self.display_name}, uuid={self.uuid}, facecount={self.facecount})"
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return False
+
+        return all(
+            getattr(self, field) == getattr(other, field) for field in ["_db", "_pk"]
+        )
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class FaceInfo:
+    """ Info about a face in the Photos library
+    """
+
+    def __init__(self, db=None, pk=None):
+        """ Creates a new FaceInfo instance
+
+        Arguments:
+            db: instance of PhotosDB object
+            pk: primary key value of face to init the object with    
+        
+        Returns:
+            FaceInfo instance
+        """
+        self._db = db
+        self._pk = pk
+
+        face = self._db._db_faceinfo_pk[pk]
+        self._info = face
+        self.uuid = face["uuid"]
+        self.name = face["fullname"]
+        self.asset_uuid = face["asset_uuid"]
+        self._person_pk = face["person"]
+        self.center_x = face["centerx"]
+        self.center_y = face["centery"]
+        self.mouth_x = face["mouthx"]
+        self.mouth_y = face["mouthy"]
+        self.left_eye_x = face["lefteyex"]
+        self.left_eye_y = face["lefteyey"]
+        self.right_eye_x = face["righteyex"]
+        self.right_eye_y = face["righteyey"]
+        self.size = face["size"]
+        self.quality = face["quality"]
+        self.source_width = face["sourcewidth"]
+        self.source_height = face["sourceheight"]
+        self.has_smile = face["has_smile"]
+        self.left_eye_closed = face["left_eye_closed"]
+        self.right_eye_closed = face["right_eye_closed"]
+        self.manual = face["manual"]
+        self.face_type = face["facetype"]
+        self.age_type = face["agetype"]
+        self.bald_type = face["baldtype"]
+        self.eye_makeup_type = face["eyemakeuptype"]
+        self.eye_state = face["eyestate"]
+        self.facial_hair_type = face["facialhairtype"]
+        self.gender_type = face["gendertype"]
+        self.glasses_type = face["glassestype"]
+        self.hair_color_type = face["haircolortype"]
+        self.intrash = face["intrash"]
+        self.lip_makeup_type = face["lipmakeuptype"]
+        self.smile_type = face["smiletype"]
+
+    @property
+    def center(self):
+        """ Coordinates, in PIL format, for center of face 
+
+        Returns:
+            tuple of coordinates in form (x, y)
+        """
+        return self._make_point((self.center_x, self.center_y))
+
+    @property
+    def size_pixels(self):
+        """ Size of face in pixels (centered around center_x, center_y)
+
+        Returns:
+            size, in int pixels, of a circle drawn around the center of the face
+        """
+        photo = self.photo
+        size_reference = photo.width if photo.width > photo.height else photo.height
+        return self.size * size_reference
+
+    @property
+    def mouth(self):
+        """ Coordinates, in PIL format, for mouth position
+
+        Returns:
+            tuple of coordinates in form (x, y)
+        """
+        return self._make_point_with_rotation((self.mouth_x, self.mouth_y))
+
+    @property
+    def left_eye(self):
+        """ Coordinates, in PIL format, for left eye position
+
+        Returns:
+            tuple of coordinates in form (x, y)
+        """
+        return self._make_point_with_rotation((self.left_eye_x, self.left_eye_y))
+
+    @property
+    def right_eye(self):
+        """ Coordinates, in PIL format, for right eye position
+
+        Returns:
+            tuple of coordinates in form (x, y)
+        """
+        return self._make_point_with_rotation((self.right_eye_x, self.right_eye_y))
+
+    @property
+    def person_info(self):
+        """ PersonInfo instance for person associated with this face """
+        try:
+            return self._person
+        except AttributeError:
+            self._person = PersonInfo(db=self._db, pk=self._person_pk)
+            return self._person
+
+    @property
+    def photo(self):
+        """ PhotoInfo instance associated with this face """
+        try:
+            return self._photo
+        except AttributeError:
+            self._photo = self._db.get_photo(self.asset_uuid)
+            if self._photo is None:
+                logging.warning(f"Could not get photo for uuid: {self.asset_uuid}")
+            return self._photo
+
+    @property
+    def mwg_rs_area(self):
+        """ Get coordinates for Metadata Working Group Region Area. 
+
+        Returns:
+            MWG_RS_Area named tuple with x, y, h, w where:
+            x = stArea:x
+            y = stArea:y
+            h = stArea:h
+            w = stArea:w
+
+        Reference:
+            https://photo.stackexchange.com/questions/106410/how-does-xmp-define-the-face-region
+        """
+        x, y = self.center_x, self.center_y
+        x, y = self._fix_orientation((x, y))
+
+        if self.photo.orientation in [5, 6, 7, 8]:
+            w = self.size_pixels / self.photo.height
+            h = self.size_pixels / self.photo.width
+        else:
+            h = self.size_pixels / self.photo.height
+            w = self.size_pixels / self.photo.width
+
+        return MWG_RS_Area(x, y, h, w)
+
+    @property
+    def mpri_reg_rect(self):
+        """ Get coordinates for Microsoft Photo Region Rectangle.
+
+        Returns:
+            MPRI_Reg_Rect named tuple with x, y, h, w where:
+            x = x coordinate of top left corner of rectangle
+            y = y coordinate of top left corner of rectangle
+            h = height of rectangle
+            w = width of rectangle
+
+        Reference:
+            https://docs.microsoft.com/en-us/windows/win32/wic/-wic-people-tagging
+        """
+        x, y = self.center_x, self.center_y
+        x, y = self._fix_orientation((x, y))
+
+        if self.photo.orientation in [5, 6, 7, 8]:
+            w = self.size_pixels / self.photo.width
+            h = self.size_pixels / self.photo.height
+            x = x - self.size_pixels / self.photo.height / 2
+            y = y - self.size_pixels / self.photo.width / 2
+        else:
+            h = self.size_pixels / self.photo.width
+            w = self.size_pixels / self.photo.height
+            x = x - self.size_pixels / self.photo.width / 2
+            y = y - self.size_pixels / self.photo.height / 2
+
+        return MPRI_Reg_Rect(x, y, h, w)
+
+    def face_rect(self):
+        """ Get face rectangle coordinates for current version of the associated image
+            If image has been edited, rectangle applies to edited version, otherwise original version
+            Coordinates in format and reference frame used by PIL
+
+        Returns:
+            list [(x0, x1), (y0, y1)] of coordinates in reference frame used by PIL
+        """
+        photo = self.photo
+        size_reference = photo.width if photo.width > photo.height else photo.height
+        radius = (self.size / 2) * size_reference
+        x, y = self._make_point((self.center_x, self.center_y))
+        x0, y0 = x - radius, y - radius
+        x1, y1 = x + radius, y + radius
+        return [(x0, y0), (x1, y1)]
+
+    def roll_pitch_yaw(self):
+        """ Roll, pitch, yaw of face in radians as tuple """
+        info = self._info
+        roll = 0 if info["roll"] is None else info["roll"]
+        pitch = 0 if info["pitch"] is None else info["pitch"]
+        yaw = 0 if info["yaw"] is None else info["yaw"]
+
+        return (roll, pitch, yaw)
+
+    @property
+    def roll(self):
+        """ Return roll angle in radians of the face region """
+        roll, _, _ = self.roll_pitch_yaw()
+        return roll
+
+    @property
+    def pitch(self):
+        """ Return pitch angle in radians of the face region """
+        _, pitch, _ = self.roll_pitch_yaw()
+        return pitch
+
+    @property
+    def yaw(self):
+        """ Return yaw angle in radians of the face region """
+        _, _, yaw = self.roll_pitch_yaw()
+        return yaw
+
+    def _fix_orientation(self, xy):
+        """ Translate an (x, y) tuple based on image orientation
+
+        Arguments:
+            xy: tuple of (x, y) coordinates for point to translate
+                in format used by Photos (percent of height/width)
+        
+        Returns:
+            (x, y) tuple of translated coordinates
+        """
+        # Reference: https://github.com/neilpa/phace/blob/7594776480505d0c389688a42099c94ac5d34f3f/cmd/phace/draw.go#L79-L94
+
+        orientation = self.photo.orientation
+        x, y = xy
+        if orientation == 1:
+            y = 1.0 - y
+        elif orientation == 2:
+            y = 1.0 - y
+            x = 1.0 - x
+        elif orientation == 3:
+            x = 1.0 - x
+        elif orientation == 4:
+            pass
+        elif orientation == 5:
+            x, y = 1.0 - y, x
+        elif orientation == 6:
+            x, y = 1.0 - y, 1.0 - x
+        elif orientation == 7:
+            x, y = y, x
+            y = 1.0 - y
+        elif orientation ==8:
+            x, y = y, x
+        else:
+            logging.warning(f"Unhandled orientation: {orientation}")
+
+        return (x, y)
+
+    def _make_point(self, xy):
+        """ Translate an (x, y) tuple based on image orientation
+            and convert to image coordinates
+
+        Arguments:
+            xy: tuple of (x, y) coordinates for point to translate
+                in format used by Photos (percent of height/width)
+        
+        Returns:
+            (x, y) tuple of translated coordinates in pixels in PIL format/reference frame
+        """
+        # Reference: https://github.com/neilpa/phace/blob/7594776480505d0c389688a42099c94ac5d34f3f/cmd/phace/draw.go#L79-L94
+
+        orientation = self.photo.orientation
+        x, y = self._fix_orientation(xy)
+        dx = self.photo.width
+        dy = self.photo.height
+        if orientation in [5, 6, 7, 8]:
+            dx, dy = dy, dx
+        return (int(x * dx), int(y * dy))
+
+    def _make_point_with_rotation(self, xy):
+        """ Translate an (x, y) tuple based on image orientation and rotation
+            and convert to image coordinates
+
+        Arguments:
+            xy: tuple of (x, y) coordinates for point to translate
+                in format used by Photos (percent of height/width)
+        
+        Returns:
+            (x, y) tuple of translated coordinates in pixels in PIL format/reference frame
+        """
+
+        # convert to image coordinates
+        x, y = self._make_point(xy)
+
+        # rotate about center
+        xmid, ymid = self.center
+        roll, _, _ = self.roll_pitch_yaw()
+        xr, yr = rotate_image_point(x, y, xmid, ymid, roll)
+
+        return (int(xr), int(yr))
+
+    def asdict(self):
+        """ Returns dict representation of class instance """
+        roll, pitch, yaw = self.roll_pitch_yaw()
+        return {
+            "_pk": self._pk,
+            "uuid": self.uuid,
+            "name": self.name,
+            "asset_uuid": self.asset_uuid,
+            "_person_pk": self._person_pk,
+            "center_x": self.center_x,
+            "center_y": self.center_y,
+            "center": self.center,
+            "mouth_x": self.mouth_x,
+            "mouth_y": self.mouth_y,
+            "mouth": self.mouth,
+            "left_eye_x": self.left_eye_x,
+            "left_eye_y": self.left_eye_y,
+            "left_eye": self.left_eye,
+            "right_eye_x": self.right_eye_x,
+            "right_eye_y": self.right_eye_y,
+            "right_eye": self.right_eye,
+            "size": self.size,
+            "face_rect": self.face_rect(),
+            "mpri_reg_rect": self.mpri_reg_rect._asdict(),
+            "mwg_rs_area": self.mwg_rs_area._asdict(),
+            "roll": roll,
+            "pitch": pitch,
+            "yaw": yaw,
+            "quality": self.quality,
+            "source_width": self.source_width,
+            "source_height": self.source_height,
+            "has_smile": self.has_smile,
+            "left_eye_closed": self.left_eye_closed,
+            "right_eye_closed": self.right_eye_closed,
+            "manual": self.manual,
+            "face_type": self.face_type,
+            "age_type": self.age_type,
+            "bald_type": self.bald_type,
+            "eye_makeup_type": self.eye_makeup_type,
+            "eye_state": self.eye_state,
+            "facial_hair_type": self.facial_hair_type,
+            "gender_type": self.gender_type,
+            "glasses_type": self.glasses_type,
+            "hair_color_type": self.hair_color_type,
+            "intrash": self.intrash,
+            "lip_makeup_type": self.lip_makeup_type,
+            "smile_type": self.smile_type,
+        }
+
+    def json(self):
+        """ Return JSON representation of FaceInfo instance """
+        return json.dumps(self.asdict())
+
+    def __str__(self):
+        return f"FaceInfo(uuid={self.uuid}, center_x={self.center_x}, center_y = {self.center_y}, size={self.size}, person={self.name}, asset_uuid={self.asset_uuid})"
+
+    def __repr__(self):
+        return f"FaceInfo(db={self._db}, pk={self._pk})"
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return False
+
+        return all(
+            getattr(self, field) == getattr(other, field) for field in ["_db", "_pk"]
+        )
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+def rotate_image_point(x, y, xmid, ymid, angle):
+    """ rotate image point about xm, ym by angle in radians
+
+    Arguments:
+        x: x coordinate of point to rotate    
+        y: y coordinate of point to rotate
+        xmid: x coordinate of center point to rotate about
+        ymid: y coordinate of center point to rotate about
+        angle: angle in radians about which to coordinate, 
+            counter-clockwise is positive
+
+    Returns:
+        tuple of rotated points (xr, yr)
+    """
+    # translate point relative to the mid point
+    x = x - xmid
+    y = y - ymid
+
+    # rotate by angle and translate back
+    # the photo coordinate system is downwards y is positive so
+    # need to adjust the rotation accordingly
+    cos_angle = math.cos(angle)
+    sin_angle = math.sin(angle)
+    xr = x * cos_angle + y * sin_angle + xmid
+    yr = -x * sin_angle + y * cos_angle + ymid
+
+    return (xr, yr)

osxphotos/photoinfo/__init__.py

@@ -6,4 +6,5 @@ PhotosDB.photos() returns a list of PhotoInfo objects
 
 from ._photoinfo_exifinfo import ExifInfo
 from ._photoinfo_export import ExportResults
+from ._photoinfo_scoreinfo import ScoreInfo
 from .photoinfo import PhotoInfo

osxphotos/photoinfo/_photoinfo_comments.py

@@ -0,0 +1,17 @@
+""" PhotoInfo methods to expose comments and likes for shared photos """ 
+
+@property
+def comments(self):
+    """ Returns list of Comment objects for any comments on the photo (sorted by date) """
+    try:
+        return self._db._db_comments_uuid[self.uuid]["comments"]
+    except:
+        return []
+
+@property
+def likes(self):
+    """ Returns list of Like objects for any likes on the photo (sorted by date) """
+    try:
+        return self._db._db_comments_uuid[self.uuid]["likes"]
+    except:
+        return []

osxphotos/photoinfo/_photoinfo_exiftool.py

@@ -5,6 +5,7 @@ import os
 
 from ..exiftool import ExifTool, get_exiftool_path
 
+
 @property
 def exiftool(self):
     """ Returns an ExifTool object for the photo
@@ -17,17 +18,17 @@ def exiftool(self):
         return self._exiftool
     except AttributeError:
         try:
-            exiftool_path = get_exiftool_path()
+            exiftool_path = self._db._exiftool_path or get_exiftool_path()
             if self.path is not None and os.path.isfile(self.path):
-                exiftool = ExifTool(self.path)
+                exiftool = ExifTool(self.path, exiftool=exiftool_path)
             else:
                 exiftool = None
-                logging.debug(f"exiftool: missing path {self.uuid}")
         except FileNotFoundError:
             # get_exiftool_path raises FileNotFoundError if exiftool not found
             exiftool = None
-            logging.warning(f"exiftool not in path; download and install from https://exiftool.org/")
+            logging.warning(
+                f"exiftool not in path; download and install from https://exiftool.org/"
+            )
 
         self._exiftool = exiftool
         return self._exiftool
-

osxphotos/photoinfo/_photoinfo_scoreinfo.py

@@ -0,0 +1,119 @@
+""" PhotoInfo methods to expose computed score info from the library """
+
+import logging
+from dataclasses import dataclass
+
+from .._constants import _PHOTOS_4_VERSION
+
+
+@dataclass(frozen=True)
+class ScoreInfo:
+    """ Computed photo score info associated with a photo from the Photos library """
+
+    overall: float
+    curation: float
+    promotion: float
+    highlight_visibility: float
+    behavioral: float
+    failure: float
+    harmonious_color: float
+    immersiveness: float
+    interaction: float
+    interesting_subject: float
+    intrusive_object_presence: float
+    lively_color: float
+    low_light: float
+    noise: float
+    pleasant_camera_tilt: float
+    pleasant_composition: float
+    pleasant_lighting: float
+    pleasant_pattern: float
+    pleasant_perspective: float
+    pleasant_post_processing: float
+    pleasant_reflection: float
+    pleasant_symmetry: float
+    sharply_focused_subject: float
+    tastefully_blurred: float
+    well_chosen_subject: float
+    well_framed_subject: float
+    well_timed_shot: float
+
+
+@property
+def score(self):
+    """ Computed score information for a photo
+
+    Returns:
+        ScoreInfo instance
+    """
+
+    if self._db._db_version <= _PHOTOS_4_VERSION:
+        logging.debug(f"score not implemented for this database version")
+        return None
+
+    try:
+        return self._scoreinfo  # pylint: disable=access-member-before-definition
+    except AttributeError:
+        try:
+            scores = self._db._db_scoreinfo_uuid[self.uuid]
+            self._scoreinfo = ScoreInfo(
+                overall=scores["overall_aesthetic"],
+                curation=scores["curation"],
+                promotion=scores["promotion"],
+                highlight_visibility=scores["highlight_visibility"],
+                behavioral=scores["behavioral"],
+                failure=scores["failure"],
+                harmonious_color=scores["harmonious_color"],
+                immersiveness=scores["immersiveness"],
+                interaction=scores["interaction"],
+                interesting_subject=scores["interesting_subject"],
+                intrusive_object_presence=scores["intrusive_object_presence"],
+                lively_color=scores["lively_color"],
+                low_light=scores["low_light"],
+                noise=scores["noise"],
+                pleasant_camera_tilt=scores["pleasant_camera_tilt"],
+                pleasant_composition=scores["pleasant_composition"],
+                pleasant_lighting=scores["pleasant_lighting"],
+                pleasant_pattern=scores["pleasant_pattern"],
+                pleasant_perspective=scores["pleasant_perspective"],
+                pleasant_post_processing=scores["pleasant_post_processing"],
+                pleasant_reflection=scores["pleasant_reflection"],
+                pleasant_symmetry=scores["pleasant_symmetry"],
+                sharply_focused_subject=scores["sharply_focused_subject"],
+                tastefully_blurred=scores["tastefully_blurred"],
+                well_chosen_subject=scores["well_chosen_subject"],
+                well_framed_subject=scores["well_framed_subject"],
+                well_timed_shot=scores["well_timed_shot"],
+            )
+            return self._scoreinfo
+        except KeyError:
+            self._scoreinfo = ScoreInfo(
+                overall=0.0,
+                curation=0.0,
+                promotion=0.0,
+                highlight_visibility=0.0,
+                behavioral=0.0,
+                failure=0.0,
+                harmonious_color=0.0,
+                immersiveness=0.0,
+                interaction=0.0,
+                interesting_subject=0.0,
+                intrusive_object_presence=0.0,
+                lively_color=0.0,
+                low_light=0.0,
+                noise=0.0,
+                pleasant_camera_tilt=0.0,
+                pleasant_composition=0.0,
+                pleasant_lighting=0.0,
+                pleasant_pattern=0.0,
+                pleasant_perspective=0.0,
+                pleasant_post_processing=0.0,
+                pleasant_reflection=0.0,
+                pleasant_symmetry=0.0,
+                sharply_focused_subject=0.0,
+                tastefully_blurred=0.0,
+                well_chosen_subject=0.0,
+                well_framed_subject=0.0,
+                well_timed_shot=0.0,
+            )
+            return self._scoreinfo

osxphotos/photoinfo/_photoinfo_searchinfo.py

@@ -1,11 +1,32 @@
 """ Methods and class for PhotoInfo exposing SearchInfo data such as labels 
     Adds the following properties to PhotoInfo (valid only for Photos 5):
         search_info: returns a SearchInfo object
+        search_info_normalized: returns a SearchInfo object with properties that produce normalized results
         labels: returns list of labels
         labels_normalized: returns list of normalized labels
 """
 
-from .._constants import _PHOTOS_4_VERSION, SEARCH_CATEGORY_LABEL
+from .._constants import (
+    _PHOTOS_4_VERSION,
+    SEARCH_CATEGORY_CITY,
+    SEARCH_CATEGORY_LABEL,
+    SEARCH_CATEGORY_NEIGHBORHOOD,
+    SEARCH_CATEGORY_PLACE_NAME,
+    SEARCH_CATEGORY_STREET,
+    SEARCH_CATEGORY_ALL_LOCALITY,
+    SEARCH_CATEGORY_COUNTRY,
+    SEARCH_CATEGORY_STATE,
+    SEARCH_CATEGORY_STATE_ABBREVIATION,
+    SEARCH_CATEGORY_BODY_OF_WATER,
+    SEARCH_CATEGORY_MONTH,
+    SEARCH_CATEGORY_YEAR,
+    SEARCH_CATEGORY_HOLIDAY,
+    SEARCH_CATEGORY_ACTIVITY,
+    SEARCH_CATEGORY_SEASON,
+    SEARCH_CATEGORY_VENUE,
+    SEARCH_CATEGORY_VENUE_TYPE,
+    SEARCH_CATEGORY_MEDIA_TYPES,
+)
 
 
 @property
@@ -24,6 +45,22 @@ def search_info(self):
         return self._search_info
 
 
+@property
+def search_info_normalized(self):
+    """ returns SearchInfo object for photo that produces normalized results
+        only valid on Photos 5, on older libraries, returns None
+    """
+    if self._db._db_version <= _PHOTOS_4_VERSION:
+        return None
+
+    # memoize SearchInfo object
+    try:
+        return self._search_info_normalized
+    except AttributeError:
+        self._search_info_normalized = SearchInfo(self, normalized=True)
+        return self._search_info_normalized
+
+
 @property
 def labels(self):
     """ returns list of labels applied to photo by Photos image categorization
@@ -43,14 +80,15 @@ def labels_normalized(self):
     if self._db._db_version <= _PHOTOS_4_VERSION:
         return []
 
-    return self.search_info.labels_normalized
+    return self.search_info_normalized.labels
 
 
 class SearchInfo:
     """ Info about search terms such as machine learning labels that Photos knows about a photo """
 
-    def __init__(self, photo):
-        """ photo: PhotoInfo object """
+    def __init__(self, photo, normalized=False):
+        """ photo: PhotoInfo object
+            normalized: if True, all properties return normalized (lower case) results """
 
         if photo._db._db_version <= _PHOTOS_4_VERSION:
             raise NotImplementedError(
@@ -58,6 +96,7 @@ class SearchInfo:
             )
 
         self._photo = photo
+        self._normalized = normalized
         self.uuid = photo.uuid
         try:
             # get search info for this UUID
@@ -69,25 +108,170 @@ class SearchInfo:
     @property
     def labels(self):
         """ return list of labels associated with Photo """
-        if self._db_searchinfo:
-            labels = [
-                rec["content_string"]
-                for rec in self._db_searchinfo
-                if rec["category"] == SEARCH_CATEGORY_LABEL
-            ]
-        else:
-            labels = []
-        return labels
+        return self._get_text_for_category(SEARCH_CATEGORY_LABEL)
 
     @property
-    def labels_normalized(self):
-        """ return list of normalized labels associated with Photo """
+    def place_names(self):
+        """ returns list of place names """
+        return self._get_text_for_category(SEARCH_CATEGORY_PLACE_NAME)
+
+    @property
+    def streets(self):
+        """ returns list of street names """
+        return self._get_text_for_category(SEARCH_CATEGORY_STREET)
+
+    @property
+    def neighborhoods(self):
+        """ returns list of neighborhoods """
+        return self._get_text_for_category(SEARCH_CATEGORY_NEIGHBORHOOD)
+
+    @property
+    def locality_names(self):
+        """ returns list of other locality names """
+        locality = []
+        for category in SEARCH_CATEGORY_ALL_LOCALITY:
+            locality += self._get_text_for_category(category)
+        return locality
+
+    @property
+    def city(self):
+        """ returns city/town """
+        city = self._get_text_for_category(SEARCH_CATEGORY_CITY)
+        return city[0] if city else ""
+
+    @property
+    def state(self):
+        """ returns state name """
+        state = self._get_text_for_category(SEARCH_CATEGORY_STATE)
+        return state[0] if state else ""
+
+    @property
+    def state_abbreviation(self):
+        """ returns state abbreviation """
+        abbrev = self._get_text_for_category(SEARCH_CATEGORY_STATE_ABBREVIATION)
+        return abbrev[0] if abbrev else ""
+
+    @property
+    def country(self):
+        """ returns country name """
+        country = self._get_text_for_category(SEARCH_CATEGORY_COUNTRY)
+        return country[0] if country else ""
+
+    @property
+    def month(self):
+        """ returns month name """
+        month = self._get_text_for_category(SEARCH_CATEGORY_MONTH)
+        return month[0] if month else ""
+
+    @property
+    def year(self):
+        """ returns year """
+        year = self._get_text_for_category(SEARCH_CATEGORY_YEAR)
+        return year[0] if year else ""
+
+    @property
+    def bodies_of_water(self):
+        """ returns list of body of water names """
+        return self._get_text_for_category(SEARCH_CATEGORY_BODY_OF_WATER)
+
+    @property
+    def holidays(self):
+        """ returns list of holiday names """
+        return self._get_text_for_category(SEARCH_CATEGORY_HOLIDAY)
+
+    @property
+    def activities(self):
+        """ returns list of activity names """
+        return self._get_text_for_category(SEARCH_CATEGORY_ACTIVITY)
+
+    @property
+    def season(self):
+        """ returns season name """
+        season = self._get_text_for_category(SEARCH_CATEGORY_SEASON)
+        return season[0] if season else ""
+
+    @property
+    def venues(self):
+        """ returns list of venue names """
+        return self._get_text_for_category(SEARCH_CATEGORY_VENUE)
+
+    @property
+    def venue_types(self):
+        """ returns list of venue types """
+        return self._get_text_for_category(SEARCH_CATEGORY_VENUE_TYPE)
+
+    @property
+    def media_types(self):
+        """ returns list of media types (photo, video, panorama, etc) """
+        types = []
+        for category in SEARCH_CATEGORY_MEDIA_TYPES:
+            types += self._get_text_for_category(category)
+        return types
+
+    @property
+    def all(self):
+        """ return all search info properties in a single list """
+        all = (
+            self.labels
+            + self.place_names
+            + self.streets
+            + self.neighborhoods
+            + self.locality_names
+            + self.bodies_of_water
+            + self.holidays
+            + self.activities
+            + self.venues
+            + self.venue_types
+            + self.media_types
+        )
+        if self.city:
+            all += [self.city]
+        if self.state:
+            all += [self.state]
+        if self.state_abbreviation:
+            all += [self.state_abbreviation]
+        if self.country:
+            all += [self.country]
+        if self.month:
+            all += [self.month]
+        if self.year:
+            all += [self.year]
+        if self.season:
+            all += [self.season]
+
+        return all
+
+    def asdict(self):
+        """ return dict of search info """
+        return {
+            "labels": self.labels,
+            "place_names": self.place_names,
+            "streets": self.streets,
+            "neighborhoods": self.neighborhoods,
+            "city": self.city,
+            "locality_names": self.locality_names,
+            "state": self.state,
+            "state_abbreviation": self.state_abbreviation,
+            "country": self.country,
+            "bodies_of_water": self.bodies_of_water,
+            "month": self.month,
+            "year": self.year,
+            "holidays": self.holidays,
+            "activities": self.activities,
+            "season": self.season,
+            "venues": self.venues,
+            "venue_types": self.venue_types,
+            "media_types": self.media_types,
+        }
+
+    def _get_text_for_category(self, category):
+        """ return list of text for a specified category ID """
         if self._db_searchinfo:
-            labels = [
-                rec["normalized_string"]
+            content = "normalized_string" if self._normalized else "content_string"
+            return [
+                rec[content]
                 for rec in self._db_searchinfo
-                if rec["category"] == SEARCH_CATEGORY_LABEL
+                if rec["category"] == category
             ]
         else:
-            labels = []
-        return labels
+            return []

osxphotos/photoinfo/template.py

@@ -1,66 +0,0 @@
-""" Custom template system for osxphotos (implemented in PhotoInfo.render_template) """
-
-# Rolled my own template system because:
-# 1. Needed to handle multiple values (e.g. album, keyword)
-# 2. Needed to handle default values if template not found
-# 3. Didn't want user to need to know python (e.g. by using Mako which is
-#    already used elsewhere in this project)
-# 4. Couldn't figure out how to do #1 and #2 with str.format()
-#
-# This code isn't elegant but it seems to work well.  PRs gladly accepted.
-
-import locale
-
-# ensure locale set to user's locale
-locale.setlocale(locale.LC_ALL, "")
-
-# Permitted substitutions (each of these returns a single value or None)
-TEMPLATE_SUBSTITUTIONS = {
-    "{name}": "Current filename of the photo",
-    "{original_name}": "Photo's original filename when imported to Photos",
-    "{title}": "Title of the photo",
-    "{descr}": "Description of the photo",
-    "{created.date}": "Photo's creation date in ISO format, e.g. '2020-03-22'",
-    "{created.year}": "4-digit year of file creation time",
-    "{created.yy}": "2-digit year of file creation time",
-    "{created.mm}": "2-digit month of the file creation time (zero padded)",
-    "{created.month}": "Month name in user's locale of the file creation time",
-    "{created.mon}": "Month abbreviation in the user's locale of the file creation time",
-    "{created.doy}": "3-digit day of year (e.g Julian day) of file creation time, starting from 1 (zero padded)",
-    "{modified.date}": "Photo's modification date in ISO format, e.g. '2020-03-22'",
-    "{modified.year}": "4-digit year of file modification time",
-    "{modified.yy}": "2-digit year of file modification time",
-    "{modified.mm}": "2-digit month of the file modification time (zero padded)",
-    "{modified.month}": "Month name in user's locale of the file modification time",
-    "{modified.mon}": "Month abbreviation in the user's locale of the file modification time",
-    "{modified.doy}": "3-digit day of year (e.g Julian day) of file modification time, starting from 1 (zero padded)",
-    "{place.name}": "Place name from the photo's reverse geolocation data, as displayed in Photos",
-    "{place.country_code}": "The ISO country code from the photo's reverse geolocation data",
-    "{place.name.country}": "Country name from the photo's reverse geolocation data",
-    "{place.name.state_province}": "State or province name from the photo's reverse geolocation data",
-    "{place.name.city}": "City or locality name from the photo's reverse geolocation data",
-    "{place.name.area_of_interest}": "Area of interest name (e.g. landmark or public place) from the photo's reverse geolocation data",
-    "{place.address}": "Postal address from the photo's reverse geolocation data, e.g. '2007 18th St NW, Washington, DC 20009, United States'",
-    "{place.address.street}": "Street part of the postal address, e.g. '2007 18th St NW'",
-    "{place.address.city}": "City part of the postal address, e.g. 'Washington'",
-    "{place.address.state_province}": "State/province part of the postal address, e.g. 'DC'",
-    "{place.address.postal_code}": "Postal code part of the postal address, e.g. '20009'",
-    "{place.address.country}": "Country name of the postal address, e.g. 'United States'",
-    "{place.address.country_code}": "ISO country code of the postal address, e.g. 'US'",
-}
-
-# Permitted multi-value substitutions (each of these returns None or 1 or more values)
-TEMPLATE_SUBSTITUTIONS_MULTI_VALUED = {
-    "{album}": "Album(s) photo is contained in",
-    "{folder_album}": "Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder",
-    "{keyword}": "Keyword(s) assigned to photo",
-    "{person}": "Person(s) / face(s) in a photo",
-    "{label}": "Image categorization label associated with a photo (Photos 5 only)",
-    "{label_normalized}": "All lower case version of 'label' (Photos 5 only)",
-}
-
-# Just the multi-valued substitution names without the braces
-MULTI_VALUE_SUBSTITUTIONS = [
-    field.replace("{", "").replace("}", "")
-    for field in TEMPLATE_SUBSTITUTIONS_MULTI_VALUED.keys()
-]

osxphotos/photosdb/__init__.py

@@ -4,3 +4,4 @@ Processes a Photos.app library database to extract information about photos
 """
 
 from .photosdb import PhotosDB
+from .photosdb_utils import get_db_version, get_db_model_version, get_model_version

osxphotos/photosdb/_photosdb_process_comments.py

@@ -0,0 +1,157 @@
+""" PhotosDB method for processing comments and likes on shared photos.
+    Do not import this module directly """
+
+import dataclasses
+import datetime
+from dataclasses import dataclass
+
+from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION, TIME_DELTA
+from ..utils import _open_sql_file, normalize_unicode
+
+
+def _process_comments(self):
+    """ load the comments and likes data from the database 
+        this is a PhotosDB method that should be imported in 
+        the PhotosDB class definition in photosdb.py
+    """
+    self._db_hashed_person_id = {}
+    self._db_comments_uuid = {}
+    if self._db_version <= _PHOTOS_4_VERSION:
+        _process_comments_4(self)
+    else:
+        _process_comments_5(self)
+
+
+@dataclass
+class CommentInfo:
+    """ Class for shared photo comments """
+
+    datetime: datetime.datetime
+    user: str
+    ismine: bool
+    text: str
+
+    def asdict(self):
+        return dataclasses.asdict(self)
+
+
+@dataclass
+class LikeInfo:
+    """ Class for shared photo likes """
+
+    datetime: datetime.datetime
+    user: str
+    ismine: bool
+
+    def asdict(self):
+        return dataclasses.asdict(self)
+
+
+# The following methods do not get imported into PhotosDB
+# but will get called by _process_comments
+def _process_comments_4(photosdb):
+    """ process comments and likes info for Photos <= 4 
+        photosdb: PhotosDB instance """
+    raise NotImplementedError(
+        f"Not implemented for database version {photosdb._db_version}."
+    )
+
+
+def _process_comments_5(photosdb):
+    """ process comments and likes info for Photos >= 5 
+        photosdb: PhotosDB instance """
+
+    db = photosdb._tmp_db
+
+    asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
+
+    (conn, cursor) = _open_sql_file(db)
+
+    results = conn.execute(
+        """
+        SELECT DISTINCT
+        ZINVITEEHASHEDPERSONID,
+        ZINVITEEFIRSTNAME,
+        ZINVITEELASTNAME,
+        ZINVITEEFULLNAME
+        FROM
+        ZCLOUDSHAREDALBUMINVITATIONRECORD
+        """
+    )
+
+    # order of results
+    # 0: ZINVITEEHASHEDPERSONID,
+    # 1: ZINVITEEFIRSTNAME,
+    # 2: ZINVITEELASTNAME,
+    # 3: ZINVITEEFULLNAME
+
+    photosdb._db_hashed_person_id = {}
+    for row in results.fetchall():
+        person_id = row[0]
+        photosdb._db_hashed_person_id[person_id] = {
+            "first_name": normalize_unicode(row[1]),
+            "last_name": normalize_unicode(row[2]),
+            "full_name": normalize_unicode(row[3]),
+        }
+
+    results = conn.execute(
+        f"""
+        SELECT 
+        {asset_table}.ZUUID, -- UUID of the photo
+        ZCLOUDSHAREDCOMMENT.ZISLIKE, -- comment is actually a "like"
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTDATE, -- date of comment
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTTEXT, -- text of comment
+        ZCLOUDSHAREDCOMMENT.ZCOMMENTERHASHEDPERSONID, -- hashed ID of person who made comment/like
+        ZCLOUDSHAREDCOMMENT.ZISMYCOMMENT -- is my (this user's) comment
+        FROM ZCLOUDSHAREDCOMMENT
+        JOIN {asset_table} ON
+        {asset_table}.Z_PK = ZCLOUDSHAREDCOMMENT.ZCOMMENTEDASSET
+        OR
+        {asset_table}.Z_PK = ZCLOUDSHAREDCOMMENT.ZLIKEDASSET
+    """
+    )
+
+    # order of results
+    # 0: ZGENERICASSET.ZUUID, -- UUID of the photo
+    # 1: ZCLOUDSHAREDCOMMENT.ZISLIKE, -- comment is actually a "like"
+    # 2: ZCLOUDSHAREDCOMMENT.ZCOMMENTDATE, -- date of comment
+    # 3: ZCLOUDSHAREDCOMMENT.ZCOMMENTTEXT, -- text of comment
+    # 4: ZCLOUDSHAREDCOMMENT.ZCOMMENTERHASHEDPERSONID, -- hashed ID of person who made comment/like
+    # 5: ZCLOUDSHAREDCOMMENT.ZISMYCOMMENT -- is my (this user's) comment
+
+    photosdb._db_comments_uuid = {}
+    for row in results:
+        uuid = row[0]
+        is_like = bool(row[1])
+        text = normalize_unicode(row[3])
+        try:
+            user_name = photosdb._db_hashed_person_id[row[4]]["full_name"]
+        except KeyError:
+            user_name = None
+
+        try:
+            dt = datetime.datetime.fromtimestamp(row[2] + TIME_DELTA)
+        except:
+            dt = datetime.datetime(1970, 1, 1)
+
+        ismine = bool(row[5])
+
+        try:
+            db_comments = photosdb._db_comments_uuid[uuid]
+        except KeyError:
+            photosdb._db_comments_uuid[uuid] = {"likes": [], "comments": []}
+            db_comments = photosdb._db_comments_uuid[uuid]
+
+        if is_like:
+            db_comments["likes"].append(LikeInfo(dt, user_name, ismine))
+        elif text:
+            db_comments["comments"].append(CommentInfo(dt, user_name, ismine, text))
+
+    # sort results
+    for uuid in photosdb._db_comments_uuid:
+        if photosdb._db_comments_uuid[uuid]["likes"]:
+            photosdb._db_comments_uuid[uuid]["likes"].sort(key=lambda x: x.datetime)
+        if photosdb._db_comments_uuid[uuid]["comments"]:
+            photosdb._db_comments_uuid[uuid]["comments"].sort(key=lambda x: x.datetime)
+
+    conn.close()

osxphotos/photosdb/_photosdb_process_exif.py

@@ -3,9 +3,9 @@
 
 import logging
 
-from .._constants import _PHOTOS_4_VERSION
+from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
 from ..utils import _db_is_locked, _debug, _open_sql_file
-
+from .photosdb_utils import get_db_version
 
 def _process_exifinfo(self):
     """ load the exif data from the database 
@@ -34,15 +34,17 @@ def _process_exifinfo_5(photosdb):
         photosdb: PhotosDB instance """
 
     db = photosdb._tmp_db
-    
+
+    asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
+        
     (conn, cursor) = _open_sql_file(db)
 
     result = conn.execute(
-        """
-        SELECT ZGENERICASSET.ZUUID, ZEXTENDEDATTRIBUTES.*
-        FROM ZGENERICASSET
+        f"""
+        SELECT {asset_table}.ZUUID, ZEXTENDEDATTRIBUTES.*
+        FROM {asset_table}
         JOIN ZEXTENDEDATTRIBUTES
-        ON ZEXTENDEDATTRIBUTES.ZASSET = ZGENERICASSET.Z_PK
+        ON ZEXTENDEDATTRIBUTES.ZASSET = {asset_table}.Z_PK
         """
     )
 
@@ -54,3 +56,5 @@ def _process_exifinfo_5(photosdb):
         if uuid in photosdb._db_exifinfo_uuid:
             logging.warning(f"duplicate exifinfo record found for uuid {uuid}")
         photosdb._db_exifinfo_uuid[uuid] = record
+
+    conn.close()

osxphotos/photosdb/_photosdb_process_faceinfo.py

@@ -0,0 +1,331 @@
+""" Methods for PhotosDB to add Photos face info 
+"""
+
+import logging
+
+from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
+from ..utils import _open_sql_file, normalize_unicode
+from .photosdb_utils import get_db_version
+
+
+"""
+    This module should be imported in the class defintion of PhotosDB in photosdb.py
+    Do not import this module directly
+    This module adds the following method to PhotosDB:
+        _process_faceinfo: process photo face info 
+
+    The following data structures are added to PhotosDB
+        self._db_faceinfo_pk: {pk: {faceinfo}}
+        self._db_faceinfo_uuid: {photo uuid: [face pk]}
+        self._db_faceinfo_person: {person_pk: [face_pk]}
+"""
+
+
+def _process_faceinfo(self):
+    """ Process face information
+    """
+
+    self._db_faceinfo_pk = {}
+    self._db_faceinfo_uuid = {}
+    self._db_faceinfo_person = {}
+
+    if self._db_version <= _PHOTOS_4_VERSION:
+        _process_faceinfo_4(self)
+    else:
+        _process_faceinfo_5(self)
+
+
+def _process_faceinfo_4(photosdb):
+    """ Process face information for Photos 4 databases
+
+    Args:
+        photosdb: an OSXPhotosDB instance
+    """
+    db = photosdb._tmp_db
+
+    (conn, cursor) = _open_sql_file(db)
+
+    result = cursor.execute(
+        """
+        SELECT
+        RKFace.modelId,
+        RKVersion.uuid, 
+        RKFace.uuid,
+        RKPerson.name,
+        RKFace.isInTrash,
+        RKFace.personId,
+        RKFace.imageModelId, 
+        RKFace.sourceWidth,
+        RKFace.sourceHeight, 
+        RKFace.centerX,
+        RKFace.centerY,
+        RKFace.size,
+        RKFace.leftEyeX, 
+        RKFace.leftEyeY,
+        RKFace.rightEyeX, 
+        RKFace.rightEyeY, 
+        RKFace.mouthX,
+        RKFace.mouthY, 
+        RKFace.hidden, 
+        RKFace.manual, 
+        RKFace.hasSmile, 
+        RKFace.isLeftEyeClosed, 
+        RKFace.isRightEyeClosed, 
+        RKFace.poseRoll, 
+        RKFace.poseYaw, 
+        RKFace.posePitch, 
+        RKFace.faceType,
+        RKFace.qualityMeasure
+        FROM
+        RKFace
+        JOIN RKPerson on RKPerson.modelId = RKFace.personId
+        JOIN RKVersion on RKVersion.modelId = RKFace.imageModelId
+        """
+    )
+
+    # 0     RKFace.modelId,
+    # 1     RKVersion.uuid,
+    # 2     RKFace.uuid,
+    # 3     RKPerson.name,
+    # 4     RKFace.isInTrash,
+    # 5     RKFace.personId,
+    # 6     RKFace.imageModelId,
+    # 7     RKFace.sourceWidth,
+    # 8     RKFace.sourceHeight,
+    # 9     RKFace.centerX,
+    # 10    RKFace.centerY,
+    # 11    RKFace.size,
+    # 12    RKFace.leftEyeX,
+    # 13    RKFace.leftEyeY,
+    # 14    RKFace.rightEyeX,
+    # 15    RKFace.rightEyeY,
+    # 16    RKFace.mouthX,
+    # 17    RKFace.mouthY,
+    # 18    RKFace.hidden,
+    # 19    RKFace.manual,
+    # 20    RKFace.hasSmile,
+    # 21    RKFace.isLeftEyeClosed,
+    # 22    RKFace.isRightEyeClosed,
+    # 23    RKFace.poseRoll,
+    # 24    RKFace.poseYaw,
+    # 25    RKFace.posePitch,
+    # 26    RKFace.faceType,
+    # 27    RKFace.qualityMeasure
+
+    for row in result:
+        modelid = row[0]
+        asset_uuid = row[1]
+        person_id = row[5]
+        face = {}
+        face["pk"] = modelid
+        face["asset_uuid"] = asset_uuid
+        face["uuid"] = row[2]
+        face["person"] = person_id
+        face["fullname"] = normalize_unicode(row[3])
+        face["sourcewidth"] = row[7]
+        face["sourceheight"] = row[8]
+        face["centerx"] = row[9]
+        face["centery"] = row[10]
+        face["size"] = row[11]
+        face["lefteyex"] = row[12]
+        face["lefteyey"] = row[13]
+        face["righteyex"] = row[14]
+        face["righteyey"] = row[15]
+        face["mouthx"] = row[16]
+        face["mouthy"] = row[17]
+        face["hidden"] = row[18]
+        face["manual"] = row[19]
+        face["has_smile"] = row[20]
+        face["left_eye_closed"] = row[21]
+        face["right_eye_closed"] = row[22]
+        face["roll"] = row[23]
+        face["yaw"] = row[24]
+        face["pitch"] = row[25]
+        face["facetype"] = row[26]
+        face["quality"] = row[27]
+
+        # Photos 5 only
+        face["agetype"] = None
+        face["baldtype"] = None
+        face["eyemakeuptype"] = None
+        face["eyestate"] = None
+        face["facialhairtype"] = None
+        face["gendertype"] = None
+        face["glassestype"] = None
+        face["haircolortype"] = None
+        face["intrash"] = None
+        face["lipmakeuptype"] = None
+        face["smiletype"] = None
+
+        photosdb._db_faceinfo_pk[modelid] = face
+
+        try:
+            photosdb._db_faceinfo_uuid[asset_uuid].append(modelid)
+        except KeyError:
+            photosdb._db_faceinfo_uuid[asset_uuid] = [modelid]
+
+        try:
+            photosdb._db_faceinfo_person[person_id].append(modelid)
+        except KeyError:
+            photosdb._db_faceinfo_person[person_id] = [modelid]
+
+    conn.close()
+
+
+def _process_faceinfo_5(photosdb):
+    """ Process face information for Photos 5 databases
+
+    Args:
+        photosdb: an OSXPhotosDB instance
+    """
+
+    db = photosdb._tmp_db
+
+    asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
+
+    (conn, cursor) = _open_sql_file(db)
+
+    result = cursor.execute(
+        f""" 
+        SELECT
+        ZDETECTEDFACE.Z_PK,
+        {asset_table}.ZUUID,
+        ZDETECTEDFACE.ZUUID,
+        ZDETECTEDFACE.ZPERSON,
+        ZPERSON.ZFULLNAME,
+        ZDETECTEDFACE.ZAGETYPE,
+        ZDETECTEDFACE.ZBALDTYPE,
+        ZDETECTEDFACE.ZEYEMAKEUPTYPE,
+        ZDETECTEDFACE.ZEYESSTATE,
+        ZDETECTEDFACE.ZFACIALHAIRTYPE,
+        ZDETECTEDFACE.ZGENDERTYPE,
+        ZDETECTEDFACE.ZGLASSESTYPE,
+        ZDETECTEDFACE.ZHAIRCOLORTYPE,
+        ZDETECTEDFACE.ZHASSMILE,
+        ZDETECTEDFACE.ZHIDDEN,
+        ZDETECTEDFACE.ZISINTRASH,
+        ZDETECTEDFACE.ZISLEFTEYECLOSED,
+        ZDETECTEDFACE.ZISRIGHTEYECLOSED,
+        ZDETECTEDFACE.ZLIPMAKEUPTYPE,
+        ZDETECTEDFACE.ZMANUAL,
+        ZDETECTEDFACE.ZQUALITYMEASURE,
+        ZDETECTEDFACE.ZSMILETYPE,
+        ZDETECTEDFACE.ZSOURCEHEIGHT,
+        ZDETECTEDFACE.ZSOURCEWIDTH,
+        ZDETECTEDFACE.ZBLURSCORE,
+        ZDETECTEDFACE.ZCENTERX,
+        ZDETECTEDFACE.ZCENTERY,
+        ZDETECTEDFACE.ZLEFTEYEX,
+        ZDETECTEDFACE.ZLEFTEYEY,
+        ZDETECTEDFACE.ZMOUTHX,
+        ZDETECTEDFACE.ZMOUTHY,
+        ZDETECTEDFACE.ZPOSEYAW,
+        ZDETECTEDFACE.ZQUALITY,
+        ZDETECTEDFACE.ZRIGHTEYEX,
+        ZDETECTEDFACE.ZRIGHTEYEY,
+        ZDETECTEDFACE.ZROLL,
+        ZDETECTEDFACE.ZSIZE,
+        ZDETECTEDFACE.ZYAW,
+        ZDETECTEDFACE.ZMASTERIDENTIFIER
+        FROM ZDETECTEDFACE
+        JOIN {asset_table} ON {asset_table}.Z_PK = ZDETECTEDFACE.ZASSET
+        JOIN ZPERSON ON ZPERSON.Z_PK = ZDETECTEDFACE.ZPERSON;
+        """
+    )
+
+    # 0    ZDETECTEDFACE.Z_PK
+    # 1    ZGENERICASSET.ZUUID,
+    # 2    ZDETECTEDFACE.ZUUID,
+    # 3    ZDETECTEDFACE.ZPERSON,
+    # 4    ZPERSON.ZFULLNAME,
+    # 5    ZDETECTEDFACE.ZAGETYPE,
+    # 6    ZDETECTEDFACE.ZBALDTYPE,
+    # 7    ZDETECTEDFACE.ZEYEMAKEUPTYPE,
+    # 8    ZDETECTEDFACE.ZEYESSTATE,
+    # 9    ZDETECTEDFACE.ZFACIALHAIRTYPE,
+    # 10   ZDETECTEDFACE.ZGENDERTYPE,
+    # 11   ZDETECTEDFACE.ZGLASSESTYPE,
+    # 12   ZDETECTEDFACE.ZHAIRCOLORTYPE,
+    # 13   ZDETECTEDFACE.ZHASSMILE,
+    # 14   ZDETECTEDFACE.ZHIDDEN,
+    # 15   ZDETECTEDFACE.ZISINTRASH,
+    # 16   ZDETECTEDFACE.ZISLEFTEYECLOSED,
+    # 17   ZDETECTEDFACE.ZISRIGHTEYECLOSED,
+    # 18   ZDETECTEDFACE.ZLIPMAKEUPTYPE,
+    # 19   ZDETECTEDFACE.ZMANUAL,
+    # 20   ZDETECTEDFACE.ZQUALITYMEASURE,
+    # 21   ZDETECTEDFACE.ZSMILETYPE,
+    # 22   ZDETECTEDFACE.ZSOURCEHEIGHT,
+    # 23   ZDETECTEDFACE.ZSOURCEWIDTH,
+    # 24   ZDETECTEDFACE.ZBLURSCORE,
+    # 25   ZDETECTEDFACE.ZCENTERX,
+    # 26   ZDETECTEDFACE.ZCENTERY,
+    # 27   ZDETECTEDFACE.ZLEFTEYEX,
+    # 28   ZDETECTEDFACE.ZLEFTEYEY,
+    # 29   ZDETECTEDFACE.ZMOUTHX,
+    # 30   ZDETECTEDFACE.ZMOUTHY,
+    # 31   ZDETECTEDFACE.ZPOSEYAW,
+    # 32   ZDETECTEDFACE.ZQUALITY,
+    # 33   ZDETECTEDFACE.ZRIGHTEYEX,
+    # 34   ZDETECTEDFACE.ZRIGHTEYEY,
+    # 35   ZDETECTEDFACE.ZROLL,
+    # 36   ZDETECTEDFACE.ZSIZE,
+    # 37   ZDETECTEDFACE.ZYAW,
+    # 38   ZDETECTEDFACE.ZMASTERIDENTIFIER
+
+    for row in result:
+        pk = row[0]
+        asset_uuid = row[1]
+        person_pk = row[3]
+        face = {}
+        face["pk"] = pk
+        face["asset_uuid"] = asset_uuid
+        face["uuid"] = row[2]
+        face["person"] = person_pk
+        face["fullname"] = normalize_unicode(row[4])
+        face["agetype"] = row[5]
+        face["baldtype"] = row[6]
+        face["eyemakeuptype"] = row[7]
+        face["eyestate"] = row[8]
+        face["facialhairtype"] = row[9]
+        face["gendertype"] = row[10]
+        face["glassestype"] = row[11]
+        face["haircolortype"] = row[12]
+        face["has_smile"] = row[13]
+        face["hidden"] = row[14]
+        face["intrash"] = row[15]
+        face["left_eye_closed"] = row[16]
+        face["right_eye_closed"] = row[17]
+        face["lipmakeuptype"] = row[18]
+        face["manual"] = row[19]
+        face["smiletype"] = row[21]
+        face["sourceheight"] = row[22]
+        face["sourcewidth"] = row[23]
+        face["facetype"] = None  # Photos 4 only
+        face["centerx"] = row[25]
+        face["centery"] = row[26]
+        face["lefteyex"] = row[27]
+        face["lefteyey"] = row[28]
+        face["mouthx"] = row[29]
+        face["mouthy"] = row[30]
+        face["quality"] = row[32]
+        face["righteyex"] = row[33]
+        face["righteyey"] = row[34]
+        face["roll"] = row[35]
+        face["size"] = row[36]
+        face["yaw"] = row[37]
+        face["pitch"] = 0.0  # not defined in Photos 5
+
+        photosdb._db_faceinfo_pk[pk] = face
+
+        try:
+            photosdb._db_faceinfo_uuid[asset_uuid].append(pk)
+        except KeyError:
+            photosdb._db_faceinfo_uuid[asset_uuid] = [pk]
+
+        try:
+            photosdb._db_faceinfo_person[person_pk].append(pk)
+        except KeyError:
+            photosdb._db_faceinfo_person[person_pk] = [pk]
+
+    conn.close()

osxphotos/photosdb/_photosdb_process_scoreinfo.py

@@ -0,0 +1,150 @@
+""" Methods for PhotosDB to add Photos 5 photo score info 
+    ref: https://simonwillison.net/2020/May/21/dogsheep-photos/
+"""
+
+import logging
+
+from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
+from ..utils import _open_sql_file
+from .photosdb_utils import get_db_version
+
+"""
+    This module should be imported in the class defintion of PhotosDB in photosdb.py
+    Do not import this module directly
+    This module adds the following method to PhotosDB:
+        _process_scoreinfo: process photo score info 
+
+    The following data structures are added to PhotosDB
+        self._db_scoreinfo_uuid
+    
+    These methods only work on Photos 5 databases.  Will print warning on earlier library versions.
+"""
+
+
+def _process_scoreinfo(self):
+    """ Process computed photo scores
+        Note: Only works on Photos version == 5.0
+    """
+
+    # _db_scoreinfo_uuid is dict in form {uuid: {score values}}
+    self._db_scoreinfo_uuid = {}
+
+    if self._db_version <= _PHOTOS_4_VERSION:
+        raise NotImplementedError(
+            f"search info not implemented for this database version"
+        )
+    else:
+        _process_scoreinfo_5(self)
+
+
+def _process_scoreinfo_5(photosdb):
+    """ Process computed photo scores for Photos 5 databases
+
+    Args:
+        photosdb: an OSXPhotosDB instance
+    """
+
+    db = photosdb._tmp_db
+
+    asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
+
+    (conn, cursor) = _open_sql_file(db)
+
+    result = cursor.execute(
+        f"""
+        SELECT 
+        {asset_table}.ZUUID,
+        {asset_table}.ZOVERALLAESTHETICSCORE,
+        {asset_table}.ZCURATIONSCORE,
+        {asset_table}.ZPROMOTIONSCORE,
+        {asset_table}.ZHIGHLIGHTVISIBILITYSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZBEHAVIORALSCORE, 
+        ZCOMPUTEDASSETATTRIBUTES.ZFAILURESCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZHARMONIOUSCOLORSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZIMMERSIVENESSSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZINTERACTIONSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZINTERESTINGSUBJECTSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZINTRUSIVEOBJECTPRESENCESCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZLIVELYCOLORSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZLOWLIGHT,
+        ZCOMPUTEDASSETATTRIBUTES.ZNOISESCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTCAMERATILTSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTCOMPOSITIONSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTLIGHTINGSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPATTERNSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPERSPECTIVESCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPOSTPROCESSINGSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTREFLECTIONSSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTSYMMETRYSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZSHARPLYFOCUSEDSUBJECTSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZTASTEFULLYBLURREDSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZWELLCHOSENSUBJECTSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZWELLFRAMEDSUBJECTSCORE,
+        ZCOMPUTEDASSETATTRIBUTES.ZWELLTIMEDSHOTSCORE
+        FROM {asset_table}
+        JOIN ZCOMPUTEDASSETATTRIBUTES ON ZCOMPUTEDASSETATTRIBUTES.ZASSET = {asset_table}.Z_PK
+        """
+    )
+
+    # 0     ZGENERICASSET.ZUUID,
+    # 1     ZGENERICASSET.ZOVERALLAESTHETICSCORE,
+    # 2     ZGENERICASSET.ZCURATIONSCORE,
+    # 3     ZGENERICASSET.ZPROMOTIONSCORE,
+    # 4     ZGENERICASSET.ZHIGHLIGHTVISIBILITYSCORE,
+    # 5     ZCOMPUTEDASSETATTRIBUTES.ZBEHAVIORALSCORE,
+    # 6     ZCOMPUTEDASSETATTRIBUTES.ZFAILURESCORE,
+    # 7     ZCOMPUTEDASSETATTRIBUTES.ZHARMONIOUSCOLORSCORE,
+    # 8     ZCOMPUTEDASSETATTRIBUTES.ZIMMERSIVENESSSCORE,
+    # 9     ZCOMPUTEDASSETATTRIBUTES.ZINTERACTIONSCORE,
+    # 10    ZCOMPUTEDASSETATTRIBUTES.ZINTERESTINGSUBJECTSCORE,
+    # 11    ZCOMPUTEDASSETATTRIBUTES.ZINTRUSIVEOBJECTPRESENCESCORE,
+    # 12    ZCOMPUTEDASSETATTRIBUTES.ZLIVELYCOLORSCORE,
+    # 13    ZCOMPUTEDASSETATTRIBUTES.ZLOWLIGHT,
+    # 14    ZCOMPUTEDASSETATTRIBUTES.ZNOISESCORE,
+    # 15    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTCAMERATILTSCORE,
+    # 16    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTCOMPOSITIONSCORE,
+    # 17    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTLIGHTINGSCORE,
+    # 18    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPATTERNSCORE,
+    # 19    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPERSPECTIVESCORE,
+    # 20    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTPOSTPROCESSINGSCORE,
+    # 21    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTREFLECTIONSSCORE,
+    # 22    ZCOMPUTEDASSETATTRIBUTES.ZPLEASANTSYMMETRYSCORE,
+    # 23    ZCOMPUTEDASSETATTRIBUTES.ZSHARPLYFOCUSEDSUBJECTSCORE,
+    # 24    ZCOMPUTEDASSETATTRIBUTES.ZTASTEFULLYBLURREDSCORE,
+    # 25    ZCOMPUTEDASSETATTRIBUTES.ZWELLCHOSENSUBJECTSCORE,
+    # 26    ZCOMPUTEDASSETATTRIBUTES.ZWELLFRAMEDSUBJECTSCORE,
+    # 27    ZCOMPUTEDASSETATTRIBUTES.ZWELLTIMEDSHOTSCORE
+
+    for row in result:
+        uuid = row[0]
+        scores = {"uuid": uuid}
+        scores["overall_aesthetic"] = row[1]
+        scores["curation"] = row[2]
+        scores["promotion"] = row[3]
+        scores["highlight_visibility"] = row[4]
+        scores["behavioral"] = row[5]
+        scores["failure"] = row[6]
+        scores["harmonious_color"] = row[7]
+        scores["immersiveness"] = row[8]
+        scores["interaction"] = row[9]
+        scores["interesting_subject"] = row[10]
+        scores["intrusive_object_presence"] = row[11]
+        scores["lively_color"] = row[12]
+        scores["low_light"] = row[13]
+        scores["noise"] = row[14]
+        scores["pleasant_camera_tilt"] = row[15]
+        scores["pleasant_composition"] = row[16]
+        scores["pleasant_lighting"] = row[17]
+        scores["pleasant_pattern"] = row[18]
+        scores["pleasant_perspective"] = row[19]
+        scores["pleasant_post_processing"] = row[20]
+        scores["pleasant_reflection"] = row[21]
+        scores["pleasant_symmetry"] = row[22]
+        scores["sharply_focused_subject"] = row[23]
+        scores["tastefully_blurred"] = row[24]
+        scores["well_chosen_subject"] = row[25]
+        scores["well_framed_subject"] = row[26]
+        scores["well_timed_shot"] = row[27]
+        photosdb._db_scoreinfo_uuid[uuid] = scores
+
+    conn.close()

osxphotos/photosdb/_photosdb_process_searchinfo.py

@@ -3,13 +3,14 @@
     ref: https://github.com/dogsheep/photos-to-sqlite/issues/16
 """
 
+from functools import lru_cache
 import logging
 import pathlib
 import uuid as uuidlib
 from pprint import pformat
 
 from .._constants import _PHOTOS_4_VERSION, SEARCH_CATEGORY_LABEL
-from ..utils import _db_is_locked, _debug, _open_sql_file
+from ..utils import _db_is_locked, _debug, _open_sql_file, normalize_unicode
 
 """
     This module should be imported in the class defintion of PhotosDB in photosdb.py
@@ -39,6 +40,19 @@ def _process_searchinfo(self):
         search terms.  In Photos 5, this is called psi.sqlite
         Note: Only works on Photos version == 5.0 """
 
+    # _db_searchinfo_uuid is dict in form {uuid : [list of associated search info records]
+    self._db_searchinfo_uuid = _db_searchinfo_uuid = {}
+
+    # _db_searchinfo_categories is dict in form {search info category id: list normalized strings for the category
+    # right now, this is mostly for debugging to easily see which search terms are in the library
+    self._db_searchinfo_categories = _db_searchinfo_categories = {}
+
+    # _db_searchinfo_labels is dict in form {normalized label: [list of photo uuids]}
+    # this serves as a reverse index from label to photos containing the label
+    # _db_searchinfo_labels_normalized is the same but with normalized (lower case) version of the label
+    self._db_searchinfo_labels = _db_searchinfo_labels = {}
+    self._db_searchinfo_labels_normalized = _db_searchinfo_labels_normalized = {}
+
     if self._db_version <= _PHOTOS_4_VERSION:
         raise NotImplementedError(
             f"search info not implemented for this database version"
@@ -46,7 +60,8 @@ def _process_searchinfo(self):
 
     search_db_path = pathlib.Path(self._dbfile).parent / "search" / "psi.sqlite"
     if not search_db_path.exists():
-        raise FileNotFoundError(f"could not find search db: {search_db_path}")
+        logging.warning(f"could not find search db: {search_db_path}")
+        return None
 
     if _db_is_locked(search_db_path):
         search_db = self._copy_db_file(search_db_path)
@@ -55,7 +70,7 @@ def _process_searchinfo(self):
 
     (conn, c) = _open_sql_file(search_db)
 
-    result = conn.execute(
+    result = c.execute(
         """
         select
         ga.rowid,
@@ -76,27 +91,33 @@ def _process_searchinfo(self):
         """
     )
 
-    # _db_searchinfo_uuid is dict in form {uuid : [list of associated search info records]
-    _db_searchinfo_uuid = {}
+    # 0: ga.rowid,
+    # 1: assets.uuid_0,
+    # 2: assets.uuid_1,
+    # 3: groups.rowid as groupid,
+    # 4: groups.category,
+    # 5: groups.owning_groupid,
+    # 6: groups.content_string,
+    # 7: groups.normalized_string,
+    # 8: groups.lookup_identifier
 
-    # _db_searchinfo_categories is dict in form {search info category id: list normalized strings for the category
-    # right now, this is mostly for debugging to easily see which search terms are in the library
-    _db_searchinfo_categories = {}
-
-    # _db_searchinfo_labels is dict in form {normalized label: [list of photo uuids]}
-    # this serves as a reverse index from label to photos containing the label
-    # _db_searchinfo_labels_normalized is the same but with normalized (lower case) version of the label
-    _db_searchinfo_labels = {}
-    _db_searchinfo_labels_normalized = {}
-
-    cols = [c[0] for c in result.description]
-    for row in result.fetchall():
-        record = dict(zip(cols, row))
-        uuid = ints_to_uuid(record["uuid_0"], record["uuid_1"])
+    for row in c:
+        uuid = ints_to_uuid(row[1], row[2])
         # strings have null character appended, so strip it
-        for key in record:
-            if isinstance(record[key], str):
-                record[key] = record[key].replace("\x00", "")
+        record = {
+            "uuid": uuid,
+            "rowid": row[0],
+            "uuid_0": row[1],
+            "uuid_1": row[2],
+            "groupid": row[3],
+            "category": row[4],
+            "owning_groupid": row[5],
+            "content_string": normalize_unicode(row[6].replace("\x00", "")),
+        }
+
+        record["normalized_string"] = normalize_unicode(row[7].replace("\x00", ""))
+        record["lookup_identifier"] = normalize_unicode(row[8].replace("\x00", ""))
+
         try:
             _db_searchinfo_uuid[uuid].append(record)
         except KeyError:
@@ -104,15 +125,11 @@ def _process_searchinfo(self):
 
         category = record["category"]
         try:
-            _db_searchinfo_categories[record["category"]].append(
-                record["normalized_string"]
-            )
+            _db_searchinfo_categories[category].append(record["normalized_string"])
         except KeyError:
-            _db_searchinfo_categories[record["category"]] = [
-                record["normalized_string"]
-            ]
+            _db_searchinfo_categories[category] = [record["normalized_string"]]
 
-        if record["category"] == SEARCH_CATEGORY_LABEL:
+        if category == SEARCH_CATEGORY_LABEL:
             label = record["content_string"]
             label_norm = record["normalized_string"]
             try:
@@ -122,11 +139,6 @@ def _process_searchinfo(self):
                 _db_searchinfo_labels[label] = [uuid]
                 _db_searchinfo_labels_normalized[label_norm] = [uuid]
 
-    self._db_searchinfo_categories = _db_searchinfo_categories
-    self._db_searchinfo_uuid = _db_searchinfo_uuid
-    self._db_searchinfo_labels = _db_searchinfo_labels
-    self._db_searchinfo_labels_normalized = _db_searchinfo_labels_normalized
-
     if _debug():
         logging.debug(
             "_db_searchinfo_categories: \n" + pformat(self._db_searchinfo_categories)
@@ -138,6 +150,8 @@ def _process_searchinfo(self):
             + pformat(self._db_searchinfo_labels_normalized)
         )
 
+    conn.close()
+
 
 @property
 def labels(self):
@@ -185,6 +199,7 @@ def labels_normalized_as_dict(self):
 # The following method is not imported into PhotosDB
 
 
+@lru_cache(maxsize=128)
 def ints_to_uuid(uuid_0, uuid_1):
     """ convert two signed ints into a UUID strings
         uuid_0, uuid_1: the two int components of an RFC 4122 UUID """

osxphotos/photosdb/photosdb_utils.py

@@ -0,0 +1,84 @@
+""" utility functions used by PhotosDB """
+
+import logging
+import plistlib
+
+from .._constants import (
+    _PHOTOS_5_MODEL_VERSION,
+    _PHOTOS_6_MODEL_VERSION,
+    _TESTED_DB_VERSIONS,
+)
+from ..utils import _open_sql_file
+
+
+def get_db_version(db_file):
+    """ Gets the Photos DB version from LiGlobals table
+
+    Args:
+        db_file: path to photos.db database file containing LiGlobals table
+
+    Returns: version as str
+    """
+
+    version = None
+
+    (conn, c) = _open_sql_file(db_file)
+
+    # get database version
+    c.execute("SELECT value from LiGlobals where LiGlobals.keyPath is 'libraryVersion'")
+    version = c.fetchone()[0]
+    conn.close()
+
+    if version not in _TESTED_DB_VERSIONS:
+        print(
+            f"WARNING: Only tested on database versions [{', '.join(_TESTED_DB_VERSIONS)}]"
+            + f" You have database version={version} which has not been tested"
+        )
+
+    return version
+
+
+def get_model_version(db_file):
+    """ Returns the database model version from Z_METADATA
+    
+    Args:
+        db_file: path to Photos.sqlite database file containing Z_METADATA table
+        
+    Returns: model version as str
+    """
+
+    version = None
+
+    (conn, c) = _open_sql_file(db_file)
+
+    # get database version
+    c.execute("SELECT MAX(Z_VERSION), Z_PLIST FROM Z_METADATA")
+    results = c.fetchone()
+
+    conn.close()
+
+    plist = plistlib.loads(results[1])
+    return plist["PLModelVersion"]
+
+
+def get_db_model_version(db_file):
+    """ Returns Photos version based on model version found in db_file
+    
+    Args:
+        db_file: path to Photos.sqlite file
+    
+    Returns: int of major Photos version number (e.g. 5 or 6).
+    If unknown model version found, logs warning and returns most current Photos version.
+    """
+
+    model_ver = get_model_version(db_file)
+    if _PHOTOS_5_MODEL_VERSION[0] <= model_ver <= _PHOTOS_5_MODEL_VERSION[1]:
+        db_ver = 5
+    elif _PHOTOS_6_MODEL_VERSION[0] <= model_ver <= _PHOTOS_6_MODEL_VERSION[1]:
+        db_ver = 6
+    else:
+        logging.warning(f"Unknown model version: {model_ver}")
+        # cross our fingers and try latest version
+        db_ver = 6
+
+    return db_ver

osxphotos/placeinfo.py

@@ -11,6 +11,9 @@ from collections import namedtuple  # pylint: disable=syntax-error
 import yaml
 from bpylist import archiver
 
+from ._constants import UNICODE_FORMAT
+from .utils import normalize_unicode
+
 # postal address information, returned by PlaceInfo.address
 PostalAddress = namedtuple(
     "PostalAddress",
@@ -76,30 +79,30 @@ class PLRevGeoLocationInfo:
         geoServiceProvider,
         postalAddress,
     ):
-        self.addressString = addressString
+        self.addressString = normalize_unicode(addressString)
         self.countryCode = countryCode
         self.mapItem = mapItem
         self.isHome = isHome
-        self.compoundNames = compoundNames
-        self.compoundSecondaryNames = compoundSecondaryNames
+        self.compoundNames = normalize_unicode(compoundNames)
+        self.compoundSecondaryNames = normalize_unicode(compoundSecondaryNames)
         self.version = version
         self.geoServiceProvider = geoServiceProvider
         self.postalAddress = postalAddress
 
     def __eq__(self, other):
-        for field in [
-            "addressString",
-            "countryCode",
-            "isHome",
-            "compoundNames",
-            "compoundSecondaryNames",
-            "version",
-            "geoServiceProvider",
-            "postalAddress",
-        ]:
-            if getattr(self, field) != getattr(other, field):
-                return False
-        return True
+        return all(
+            getattr(self, field) == getattr(other, field)
+            for field in [
+                "addressString",
+                "countryCode",
+                "isHome",
+                "compoundNames",
+                "compoundSecondaryNames",
+                "version",
+                "geoServiceProvider",
+                "postalAddress",
+            ]
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -151,21 +154,17 @@ class PLRevGeoMapItem:
         self.finalPlaceInfos = finalPlaceInfos
 
     def __eq__(self, other):
-        for field in ["sortedPlaceInfos", "finalPlaceInfos"]:
-            if getattr(self, field) != getattr(other, field):
-                return False
-        return True
+        return all(
+            getattr(self, field) == getattr(other, field)
+            for field in ["sortedPlaceInfos", "finalPlaceInfos"]
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     def __str__(self):
-        sortedPlaceInfos = []
-        finalPlaceInfos = []
-        for place in self.sortedPlaceInfos:
-            sortedPlaceInfos.append(str(place))
-        for place in self.finalPlaceInfos:
-            finalPlaceInfos.append(str(place))
+        sortedPlaceInfos = [str(place) for place in self.sortedPlaceInfos]
+        finalPlaceInfos = [str(place) for place in self.finalPlaceInfos]
         return (
             f"finalPlaceInfos: {finalPlaceInfos}, sortedPlaceInfos: {sortedPlaceInfos}"
         )
@@ -187,15 +186,15 @@ class PLRevGeoMapItemAdditionalPlaceInfo:
 
     def __init__(self, area, name, placeType, dominantOrderType):
         self.area = area
-        self.name = name
+        self.name = normalize_unicode(name)
         self.placeType = placeType
         self.dominantOrderType = dominantOrderType
 
     def __eq__(self, other):
-        for field in ["area", "name", "placeType", "dominantOrderType"]:
-            if getattr(self, field) != getattr(other, field):
-                return False
-        return True
+        return all(
+            getattr(self, field) == getattr(other, field)
+            for field in ["area", "name", "placeType", "dominantOrderType"]
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -236,28 +235,28 @@ class CNPostalAddress:
         _subLocality,
     ):
         self._ISOCountryCode = _ISOCountryCode
-        self._city = _city
-        self._country = _country
-        self._postalCode = _postalCode
-        self._state = _state
-        self._street = _street
-        self._subAdministrativeArea = _subAdministrativeArea
-        self._subLocality = _subLocality
+        self._city = normalize_unicode(_city)
+        self._country = normalize_unicode(_country)
+        self._postalCode = normalize_unicode(_postalCode)
+        self._state = normalize_unicode(_state)
+        self._street = normalize_unicode(_street)
+        self._subAdministrativeArea = normalize_unicode(_subAdministrativeArea)
+        self._subLocality = normalize_unicode(_subLocality)
 
     def __eq__(self, other):
-        for field in [
-            "_ISOCountryCode",
-            "_city",
-            "_country",
-            "_postalCode",
-            "_state",
-            "_street",
-            "_subAdministrativeArea",
-            "_subLocality",
-        ]:
-            if getattr(self, field) != getattr(other, field):
-                return False
-        return True
+        return all(
+            getattr(self, field) == getattr(other, field)
+            for field in [
+                "_ISOCountryCode",
+                "_city",
+                "_country",
+                "_postalCode",
+                "_state",
+                "_street",
+                "_subAdministrativeArea",
+                "_subLocality",
+            ]
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -418,9 +417,9 @@ class PlaceInfo4(PlaceInfo):
             #   2: type
             #   3: area
             try:
-                places_dict[p[2]].append((p[1], p[3]))
+                places_dict[p[2]].append((normalize_unicode(p[1]), p[3]))
             except KeyError:
-                places_dict[p[2]] = [(p[1], p[3])]
+                places_dict[p[2]] = [(normalize_unicode(p[1]), p[3])]
 
         # build list to populate PlaceNames tuple
         # initialize with empty lists for each field in PlaceNames
@@ -490,16 +489,14 @@ class PlaceInfo4(PlaceInfo):
             "names": self.names,
             "country_code": self.country_code,
         }
-        strval = "PlaceInfo(" + ", ".join([f"{k}='{v}'" for k, v in info.items()]) + ")"
-        return strval
+        return "PlaceInfo(" + ", ".join([f"{k}='{v}'" for k, v in info.items()]) + ")"
 
-    def as_dict(self):
-        info = {
+    def asdict(self):
+        return {
             "name": self.name,
             "names": self.names._asdict(),
             "country_code": self.country_code,
         }
-        return info
 
 
 class PlaceInfo5(PlaceInfo):
@@ -509,7 +506,6 @@ class PlaceInfo5(PlaceInfo):
         """ revgeoloc_bplist: a binary plist blob containing 
                 a serialized PLRevGeoLocationInfo object """
         self._bplist = revgeoloc_bplist
-        # todo: check for None?
         self._plrevgeoloc = archiver.unarchive(revgeoloc_bplist)
         self._process_place_info()
 
@@ -541,17 +537,23 @@ class PlaceInfo5(PlaceInfo):
     @property
     def address(self):
         addr = self._plrevgeoloc.postalAddress
-        address = PostalAddress(
-            street=addr._street,
-            sub_locality=addr._subLocality,
-            city=addr._city,
-            sub_administrative_area=addr._subAdministrativeArea,
-            state_province=addr._state,
-            postal_code=addr._postalCode,
-            country=addr._country,
-            iso_country_code=addr._ISOCountryCode,
-        )
-        return address
+        if addr is not None:
+            postal_address = PostalAddress(
+                street=addr._street,
+                sub_locality=addr._subLocality,
+                city=addr._city,
+                sub_administrative_area=addr._subAdministrativeArea,
+                state_province=addr._state,
+                postal_code=addr._postalCode,
+                country=addr._country,
+                iso_country_code=addr._ISOCountryCode,
+            )
+        else:
+            postal_address = PostalAddress(
+                None, None, None, None, None, None, None, None
+            )
+
+        return postal_address
 
     def _process_place_info(self):
         """ Process sortedPlaceInfos to set self._name and self._names """
@@ -630,16 +632,14 @@ class PlaceInfo5(PlaceInfo):
             "address_str": self.address_str,
             "address": str(self.address),
         }
-        strval = "PlaceInfo(" + ", ".join([f"{k}='{v}'" for k, v in info.items()]) + ")"
-        return strval
+        return "PlaceInfo(" + ", ".join([f"{k}='{v}'" for k, v in info.items()]) + ")"
 
-    def as_dict(self):
-        info = {
+    def asdict(self):
+        return {
             "name": self.name,
             "names": self.names._asdict(),
             "country_code": self.country_code,
             "ishome": self.ishome,
             "address_str": self.address_str,
-            "address": self.address._asdict(),
+            "address": self.address._asdict() if self.address is not None else None,
         }
-        return info

osxphotos/templates/DESCRIPTION.txt

@@ -0,0 +1,8 @@
+Templates for creating XMP sidecar files
+
+* xmp_sidecar.mako: Mako template for generating XMP sidecar
+* xmp_sidecar_beta.mako: template for beta testing new XMP properties, if run with --beta, osxphotos will use this template, otherwise it always uses xmp_sidecar.mako
+
+Reference: 
+* https://www.adobe.com/devnet/xmp.html
+* https://www.makotemplates.org/

osxphotos/templates/xmp_sidecar.mako

@@ -1,30 +1,51 @@
-<!-- Created with osxphotos https://github.com/RhetTbull/osxphotos -->
+<%def name="photoshop_sidecar_for_extension(extension)">
+    % if extension is None:
+        <photoshop:SidecarForExtension></photoshop:SidecarForExtension>
+    % else:
+        <photoshop:SidecarForExtension>${extension}</photoshop:SidecarForExtension>
+    % endif
+</%def>
 
 <%def name="dc_description(desc)">
     % if desc is None:
-        <dc:description></dc:description>
+        <dc:description>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'/>
+         </rdf:Alt>
+        </dc:description>
     % else:
-        <dc:description>${desc}</dc:description>
+        <dc:description>
+        <rdf:Alt>
+         <rdf:li xml:lang='x-default'>${desc | x}</rdf:li>
+        </rdf:Alt>
+        </dc:description>
     % endif
 </%def>
 
 <%def name="dc_title(title)">
     % if title is None:
-        <dc:title></dc:title>
+        <dc:title>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'/>
+         </rdf:Alt>
+        </dc:title>
     % else:
-        <dc:title>${title}</dc:title>
+        <dc:title>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'>${title | x}</rdf:li>
+         </rdf:Alt>
+        </dc:title>
     % endif
 </%def>
 
 <%def name="dc_subject(subject)">
     % if subject:
-        <!-- keywords and persons listed in <dc:subject> as Photos does -->
         <dc:subject>
-            <rdf:Seq>
-            % for subj in subject:
-                <rdf:li>${subj}</rdf:li>
-            % endfor
-            </rdf:Seq>
+         <rdf:Bag>
+         % for subj in subject:
+          <rdf:li>${subj | x}</rdf:li>
+         % endfor
+         </rdf:Bag>
         </dc:subject>
     % endif
 </%def>
@@ -38,11 +59,11 @@
 <%def name="iptc_personinimage(persons)">
     % if persons:
         <Iptc4xmpExt:PersonInImage>
-            <rdf:Bag>
-                % for person in persons:
-                    <rdf:li>${person}</rdf:li>
-                % endfor
-            </rdf:Bag>
+        <rdf:Bag>
+         % for person in persons:
+         <rdf:li>${person | x}</rdf:li>
+         % endfor
+        </rdf:Bag>
         </Iptc4xmpExt:PersonInImage>
     % endif 
 </%def>
@@ -50,11 +71,11 @@
 <%def name="dk_tagslist(keywords)">
     % if keywords:
         <digiKam:TagsList>
-            <rdf:Seq>
-            % for keyword in keywords:
-                <rdf:li>${keyword}</rdf:li>
-            % endfor
-            </rdf:Seq>
+        <rdf:Seq>
+         % for keyword in keywords:
+          <rdf:li>${keyword | x}</rdf:li>
+         % endfor
+        </rdf:Seq>
         </digiKam:TagsList>
     % endif
 </%def>
@@ -71,29 +92,119 @@
     % endif
 </%def>
 
-<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="XMP Core 5.4.0">
-    <!-- mirrors Photos 5 "Export IPTC as XMP" option -->
-    <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
-        <rdf:Description rdf:about="" 
-            xmlns:dc="http://purl.org/dc/elements/1.1/" 
-            xmlns:photoshop="http://ns.adobe.com/photoshop/1.0/">
-            ${dc_description(photo.description)}
-            ${dc_title(photo.title)}
-            ${dc_subject(subjects)}
-            ${dc_datecreated(photo.date)}
-        </rdf:Description>
-        <rdf:Description rdf:about='' 
-            xmlns:Iptc4xmpExt='http://iptc.org/std/Iptc4xmpExt/2008-02-29/'>
-            ${iptc_personinimage(persons)}
-        </rdf:Description>
-        <rdf:Description rdf:about='' 
-            xmlns:digiKam='http://www.digikam.org/ns/1.0/'>
-            ${dk_tagslist(keywords)}
-        </rdf:Description>
-        <rdf:Description rdf:about='' 
-            xmlns:xmp='http://ns.adobe.com/xap/1.0/'>
-            ${adobe_createdate(photo.date)}
-            ${adobe_modifydate(photo.date)}
-        </rdf:Description>
-   </rdf:RDF>
-</x:xmpmeta>
+<%def name="gps_info(latitude, longitude)">
+    % if latitude is not None and longitude is not None:
+        <exif:GPSLongitude>${int(abs(longitude))},${(abs(longitude) % 1) * 60}${"E" if longitude >= 0 else "W"}</exif:GPSLongitude>
+        <exif:GPSLatitude>${int(abs(latitude))},${(abs(latitude) % 1) * 60}${"N" if latitude >= 0 else "S"}</exif:GPSLatitude>
+    % endif
+</%def>
+
+<%def name="orientation(orientation)">
+    % if orientation is not None:
+        <tiff:Orientation>${orientation}</tiff:Orientation>
+    % endif
+</%def>
+
+<%def name="mwg_face_regions(photo)">
+    % if photo.face_info:
+    <mwg-rs:Regions rdf:parseType="Resource">
+    <mwg-rs:AppliedToDimensions rdf:parseType="Resource">
+    <stDim:unit>pixel</stDim:unit>
+    </mwg-rs:AppliedToDimensions>
+    <mwg-rs:RegionList>
+    <rdf:Bag>
+    % for face in photo.face_info:
+     <rdf:li rdf:parseType="Resource">
+      <mwg-rs:Area rdf:parseType="Resource">
+      <stArea:h>${'{0:.6f}'.format(face.mwg_rs_area.h)}</stArea:h>
+      <stArea:w>${'{0:.6f}'.format(face.mwg_rs_area.w)}</stArea:w>
+      <stArea:x>${'{0:.6f}'.format(face.mwg_rs_area.x)}</stArea:x>
+      <stArea:y>${'{0:.6f}'.format(face.mwg_rs_area.y)}</stArea:y>
+      <stArea:unit>normalized</stArea:unit>
+      </mwg-rs:Area>
+      <mwg-rs:Name>${face.name}</mwg-rs:Name>
+      <mwg-rs:Rotation>${face.roll}</mwg-rs:Rotation>
+      <mwg-rs:Type>Face</mwg-rs:Type>
+     </rdf:li>
+    % endfor
+    </rdf:Bag>
+    </mwg-rs:RegionList>
+    </mwg-rs:Regions>
+    % endif
+</%def>
+
+<%def name="mpri_face_regions(photo)">
+    % if photo.face_info:
+    <MP:RegionInfo rdf:parseType="Resource">
+     <MPRI:Regions>
+      <rdf:Bag>
+      % for face in photo.face_info:
+       <rdf:li rdf:parseType="Resource">
+       <MPReg:PersonDisplayName>${face.name}</MPReg:PersonDisplayName>
+       <MPReg:Rectangle>${'{0:.6f}'.format(face.mpri_reg_rect.x)}, ${'{0:.6f}'.format(face.mpri_reg_rect.y)}, ${'{0:.6f}'.format(face.mpri_reg_rect.h)}, ${'{0:.6f}'.format(face.mpri_reg_rect.w)}</MPReg:Rectangle>
+       </rdf:li>
+      % endfor
+      </rdf:Bag>
+     </MPRI:Regions>
+    </MP:RegionInfo>
+    % endif
+</%def>
+
+
+<?xpacket begin="${"\uFEFF"}" id="W5M0MpCehiHzreSzNTczkc9d"?>
+<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="osxphotos ${version}">
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+<rdf:Description rdf:about="" 
+ xmlns:dc="http://purl.org/dc/elements/1.1/" 
+ xmlns:photoshop="http://ns.adobe.com/photoshop/1.0/">
+ ${photoshop_sidecar_for_extension(extension)}
+ ${dc_description(description)}
+ ${dc_title(photo.title)}
+ ${dc_subject(subjects)}
+ ${dc_datecreated(photo.date)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""  
+ xmlns:Iptc4xmpExt='http://iptc.org/std/Iptc4xmpExt/2008-02-29/'>
+ ${iptc_personinimage(persons)}
+</rdf:Description>
+
+<rdf:Description rdf:about="" 
+ xmlns:digiKam='http://www.digikam.org/ns/1.0/'>
+ ${dk_tagslist(keywords)}
+</rdf:Description>
+
+<rdf:Description rdf:about="" 
+ xmlns:xmp='http://ns.adobe.com/xap/1.0/'>
+ ${adobe_createdate(photo.date)}
+ ${adobe_modifydate(photo.date)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""
+ xmlns:exif='http://ns.adobe.com/exif/1.0/'>
+ ${gps_info(*photo.location)}
+</rdf:Description>
+
+<rdf:Description rdf:about=''
+ xmlns:tiff='http://ns.adobe.com/tiff/1.0/'>
+ ${orientation(photo.orientation)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""
+ xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
+ xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#"
+ xmlns:stDim="http://ns.adobe.com/xap/1.0/sType/Dimensions#">
+ ${mwg_face_regions(photo)}
+</rdf:Description>
+
+
+<rdf:Description rdf:about=""
+ xmlns:MP="http://ns.microsoft.com/photo/1.2/"
+ xmlns:MPRI="http://ns.microsoft.com/photo/1.2/t/RegionInfo#"
+ xmlns:MPReg="http://ns.microsoft.com/photo/1.2/t/Region#">
+ ${mpri_face_regions(photo)}
+</rdf:Description>
+
+</rdf:RDF>
+</x:xmpmeta>
+<?xpacket end="w"?>

osxphotos/templates/xmp_sidecar_beta.mako

@@ -0,0 +1,210 @@
+<%def name="photoshop_sidecar_for_extension(extension)">
+    % if extension is None:
+        <photoshop:SidecarForExtension></photoshop:SidecarForExtension>
+    % else:
+        <photoshop:SidecarForExtension>${extension}</photoshop:SidecarForExtension>
+    % endif
+</%def>
+
+<%def name="dc_description(desc)">
+    % if desc is None:
+        <dc:description>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'/>
+         </rdf:Alt>
+        </dc:description>
+    % else:
+        <dc:description>
+        <rdf:Alt>
+         <rdf:li xml:lang='x-default'>${desc | x}</rdf:li>
+        </rdf:Alt>
+        </dc:description>
+    % endif
+</%def>
+
+<%def name="dc_title(title)">
+    % if title is None:
+        <dc:title>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'/>
+         </rdf:Alt>
+        </dc:title>
+    % else:
+        <dc:title>
+         <rdf:Alt>
+          <rdf:li xml:lang='x-default'>${title | x}</rdf:li>
+         </rdf:Alt>
+        </dc:title>
+    % endif
+</%def>
+
+<%def name="dc_subject(subject)">
+    % if subject:
+        <dc:subject>
+         <rdf:Bag>
+         % for subj in subject:
+          <rdf:li>${subj | x}</rdf:li>
+         % endfor
+         </rdf:Bag>
+        </dc:subject>
+    % endif
+</%def>
+
+<%def name="dc_datecreated(date)">
+    % if date is not None:
+        <photoshop:DateCreated>${date.isoformat()}</photoshop:DateCreated>
+    % endif
+</%def>
+
+<%def name="iptc_personinimage(persons)">
+    % if persons:
+        <Iptc4xmpExt:PersonInImage>
+        <rdf:Bag>
+         % for person in persons:
+         <rdf:li>${person | x}</rdf:li>
+         % endfor
+        </rdf:Bag>
+        </Iptc4xmpExt:PersonInImage>
+    % endif 
+</%def>
+
+<%def name="dk_tagslist(keywords)">
+    % if keywords:
+        <digiKam:TagsList>
+        <rdf:Seq>
+         % for keyword in keywords:
+          <rdf:li>${keyword | x}</rdf:li>
+         % endfor
+        </rdf:Seq>
+        </digiKam:TagsList>
+    % endif
+</%def>
+
+<%def name="adobe_createdate(date)">
+    % if date is not None:
+        <xmp:CreateDate>${date.strftime("%Y-%m-%dT%H:%M:%S")}</xmp:CreateDate>
+    % endif
+</%def>
+
+<%def name="adobe_modifydate(date)">
+    % if date is not None:
+        <xmp:ModifyDate>${date.strftime("%Y-%m-%dT%H:%M:%S")}</xmp:ModifyDate>
+    % endif
+</%def>
+
+<%def name="gps_info(latitude, longitude)">
+    % if latitude is not None and longitude is not None:
+        <exif:GPSLongitude>${int(abs(longitude))},${(abs(longitude) % 1) * 60}${"E" if longitude >= 0 else "W"}</exif:GPSLongitude>
+        <exif:GPSLatitude>${int(abs(latitude))},${(abs(latitude) % 1) * 60}${"N" if latitude >= 0 else "S"}</exif:GPSLatitude>
+    % endif
+</%def>
+
+<%def name="orientation(orientation)">
+    % if orientation is not None:
+        <tiff:Orientation>${orientation}</tiff:Orientation>
+    % endif
+</%def>
+
+<%def name="mwg_face_regions(photo)">
+    % if photo.face_info:
+    <mwg-rs:Regions rdf:parseType="Resource">
+    <mwg-rs:AppliedToDimensions rdf:parseType="Resource">
+    <stDim:unit>pixel</stDim:unit>
+    </mwg-rs:AppliedToDimensions>
+    <mwg-rs:RegionList>
+    <rdf:Bag>
+    % for face in photo.face_info:
+     <rdf:li rdf:parseType="Resource">
+      <mwg-rs:Area rdf:parseType="Resource">
+      <stArea:h>${'{0:.6f}'.format(face.mwg_rs_area.h)}</stArea:h>
+      <stArea:w>${'{0:.6f}'.format(face.mwg_rs_area.w)}</stArea:w>
+      <stArea:x>${'{0:.6f}'.format(face.mwg_rs_area.x)}</stArea:x>
+      <stArea:y>${'{0:.6f}'.format(face.mwg_rs_area.y)}</stArea:y>
+      <stArea:unit>normalized</stArea:unit>
+      </mwg-rs:Area>
+      <mwg-rs:Name>${face.name}</mwg-rs:Name>
+      <mwg-rs:Rotation>${face.roll}</mwg-rs:Rotation>
+      <mwg-rs:Type>Face</mwg-rs:Type>
+     </rdf:li>
+    % endfor
+    </rdf:Bag>
+    </mwg-rs:RegionList>
+    </mwg-rs:Regions>
+    % endif
+</%def>
+
+<%def name="mpri_face_regions(photo)">
+    % if photo.face_info:
+    <MP:RegionInfo rdf:parseType="Resource">
+     <MPRI:Regions>
+      <rdf:Bag>
+      % for face in photo.face_info:
+       <rdf:li rdf:parseType="Resource">
+       <MPReg:PersonDisplayName>${face.name}</MPReg:PersonDisplayName>
+       <MPReg:Rectangle>${'{0:.6f}'.format(face.mpri_reg_rect.x)}, ${'{0:.6f}'.format(face.mpri_reg_rect.y)}, ${'{0:.6f}'.format(face.mpri_reg_rect.h)}, ${'{0:.6f}'.format(face.mpri_reg_rect.w)}</MPReg:Rectangle>
+       </rdf:li>
+      % endfor
+      </rdf:Bag>
+     </MPRI:Regions>
+    </MP:RegionInfo>
+    % endif
+</%def>
+
+
+<?xpacket begin="${"\uFEFF"}" id="W5M0MpCehiHzreSzNTczkc9d"?>
+<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="osxphotos ${version}">
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+<rdf:Description rdf:about="" 
+ xmlns:dc="http://purl.org/dc/elements/1.1/" 
+ xmlns:photoshop="http://ns.adobe.com/photoshop/1.0/">
+ ${photoshop_sidecar_for_extension(extension)}
+ ${dc_description(description)}
+ ${dc_title(photo.title)}
+ ${dc_subject(subjects)}
+ ${dc_datecreated(photo.date)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""  
+ xmlns:Iptc4xmpExt='http://iptc.org/std/Iptc4xmpExt/2008-02-29/'>
+ ${iptc_personinimage(persons)}
+</rdf:Description>
+
+<rdf:Description rdf:about="" 
+ xmlns:digiKam='http://www.digikam.org/ns/1.0/'>
+ ${dk_tagslist(keywords)}
+</rdf:Description>
+
+<rdf:Description rdf:about="" 
+ xmlns:xmp='http://ns.adobe.com/xap/1.0/'>
+ ${adobe_createdate(photo.date)}
+ ${adobe_modifydate(photo.date)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""
+ xmlns:exif='http://ns.adobe.com/exif/1.0/'>
+ ${gps_info(*photo.location)}
+</rdf:Description>
+
+<rdf:Description rdf:about=''
+ xmlns:tiff='http://ns.adobe.com/tiff/1.0/'>
+ ${orientation(photo.orientation)}
+</rdf:Description>
+
+<rdf:Description rdf:about=""
+ xmlns:mwg-rs="http://www.metadataworkinggroup.com/schemas/regions/"
+ xmlns:stArea="http://ns.adobe.com/xmp/sType/Area#"
+ xmlns:stDim="http://ns.adobe.com/xap/1.0/sType/Dimensions#">
+ ${mwg_face_regions(photo)}
+</rdf:Description>
+
+
+<rdf:Description rdf:about=""
+ xmlns:MP="http://ns.microsoft.com/photo/1.2/"
+ xmlns:MPRI="http://ns.microsoft.com/photo/1.2/t/RegionInfo#"
+ xmlns:MPReg="http://ns.microsoft.com/photo/1.2/t/Region#">
+ ${mpri_face_regions(photo)}
+</rdf:Description>
+
+</rdf:RDF>
+</x:xmpmeta>
+<?xpacket end="w"?>

osxphotos/utils.py

@@ -1,5 +1,6 @@
 import fnmatch
 import glob
+import inspect
 import logging
 import os
 import os.path
@@ -9,16 +10,15 @@ import re
 import sqlite3
 import subprocess
 import sys
-import tempfile
+import unicodedata
 import urllib.parse
 from plistlib import load as plistload
 
 import CoreFoundation
 import CoreServices
 import objc
-from Foundation import *
 
-from osxphotos._applescript import AppleScript
+from ._constants import UNICODE_FORMAT
 
 _DEBUG = False
 
@@ -56,9 +56,22 @@ def _debug():
     return _DEBUG
 
 
+def noop(*args, **kwargs):
+    """ do nothing (no operation) """
+    pass
+
+
+def lineno(filename):
+    """ Returns string with filename and current line number in caller as '(filename): line_num'
+    Will trim filename to just the name, dropping path, if any. """
+    line = inspect.currentframe().f_back.f_lineno
+    filename = pathlib.Path(filename).name
+    return f"{filename}: {line}"
+
+
 def _get_os_version():
-    # returns tuple containing OS version
-    # e.g. 10.13.6 = (10, 13, 6)
+    # returns tuple of str containing OS version
+    # e.g. 10.13.6 = ("10", "13", "6")
     version = platform.mac_ver()[0].split(".")
     if len(version) == 2:
         (ver, major) = version
@@ -118,60 +131,6 @@ def _dd_to_dms(dd):
     return int(deg_), int(min_), sec_
 
 
-def _hardlink_file(src, dest):
-    """ Hardlinks a file from src path to dest path 
-        src: source path as string 
-        dest: destination path as string
-        Raises exception if linking fails or either path is None """
-
-    if src is None or dest is None:
-        raise ValueError("src and dest must not be None", src, dest)
-
-    if not os.path.isfile(src):
-        raise FileNotFoundError("src file does not appear to exist", src)
-
-    # if error on copy, subprocess will raise CalledProcessError
-    try:
-        os.link(src, dest)
-    except Exception as e:
-        logging.critical(f"os.link returned error: {e}")
-        raise e
-
-
-def _copy_file(src, dest, norsrc=False):
-    """ Copies a file from src path to dest path 
-        src: source path as string 
-        dest: destination path as string
-        norsrc: (bool) if True, uses --norsrc flag with ditto so it will not copy
-                resource fork or extended attributes.  May be useful on volumes that
-                don't work with extended attributes (likely only certain SMB mounts)
-                default is False
-        Uses ditto to perform copy; will silently overwrite dest if it exists
-        Raises exception if copy fails or either path is None """
-
-    if src is None or dest is None:
-        raise ValueError("src and dest must not be None", src, dest)
-
-    if not os.path.isfile(src):
-        raise FileNotFoundError("src file does not appear to exist", src)
-
-    if norsrc:
-        command = ["/usr/bin/ditto", "--norsrc", src, dest]
-    else:
-        command = ["/usr/bin/ditto", src, dest]
-
-    # if error on copy, subprocess will raise CalledProcessError
-    try:
-        result = subprocess.run(command, check=True, stderr=subprocess.PIPE)
-    except subprocess.CalledProcessError as e:
-        logging.critical(
-            f"ditto returned error: {e.returncode} {e.stderr.decode(sys.getfilesystemencoding()).rstrip()}"
-        )
-        raise e
-
-    return result.returncode
-
-
 def dd_to_dms_str(lat, lon):
     """ convert latitude, longitude in degrees to degrees, minutes, seconds as string """
     """ lat: latitude in degrees  """
@@ -203,7 +162,7 @@ def dd_to_dms_str(lat, lon):
 
 def get_system_library_path():
     """ return the path to the system Photos library as string """
-    """ only works on MacOS 10.15+ """
+    """ only works on MacOS 10.15 """
     """ on earlier versions, returns None """
     _, major, _ = _get_os_version()
     if int(major) < 15:
@@ -220,16 +179,10 @@ def get_system_library_path():
         with open(plist_file, "rb") as fp:
             pl = plistload(fp)
     else:
-        logging.warning(f"could not find plist file: {str(plist_file)}")
+        logging.debug(f"could not find plist file: {str(plist_file)}")
         return None
 
-    photospath = pl["SystemLibraryPath"]
-
-    if photospath is not None:
-        return photospath
-    else:
-        logging.warning("Could not get path to Photos database")
-        return None
+    return pl.get("SystemLibraryPath")
 
 
 def get_last_library_path():
@@ -248,14 +201,14 @@ def get_last_library_path():
 
     # get the IPXDefaultLibraryURLBookmark from com.apple.Photos.plist
     # this is a serialized CFData object
-    photosurlref = pl["IPXDefaultLibraryURLBookmark"]
+    photosurlref = pl.get("IPXDefaultLibraryURLBookmark")
 
     if photosurlref is not None:
         # use CFURLCreateByResolvingBookmarkData to de-serialize bookmark data into a CFURLRef
         # pylint: disable=no-member
         # pylint: disable=undefined-variable
         photosurl = CoreFoundation.CFURLCreateByResolvingBookmarkData(
-            kCFAllocatorDefault, photosurlref, 0, None, None, None, None
+            CoreFoundation.kCFAllocatorDefault, photosurlref, 0, None, None, None, None
         )
 
         # the CFURLRef we got is a sruct that python treats as an array
@@ -308,40 +261,24 @@ def list_photo_libraries():
     return lib_list
 
 
-def create_path_by_date(dest, dt):
-    """ Creates a path in dest folder in form dest/YYYY/MM/DD/
-        dest: valid path as str
-        dt: datetime.timetuple() object
-        Checks to see if path exists, if it does, do nothing and return path
-        If path does not exist, creates it and returns path"""
-    if not os.path.isdir(dest):
-        raise FileNotFoundError(f"dest {dest} must be valid path")
-    yyyy, mm, dd = dt[0:3]
-    yyyy = str(yyyy).zfill(4)
-    mm = str(mm).zfill(2)
-    dd = str(dd).zfill(2)
-    new_dest = os.path.join(dest, yyyy, mm, dd)
-    if not os.path.isdir(new_dest):
-        os.makedirs(new_dest)
-    return new_dest
-
-
 def get_preferred_uti_extension(uti):
     """ get preferred extension for a UTI type
         uti: UTI str, e.g. 'public.jpeg'
         returns: preferred extension as str """
 
     # reference: https://developer.apple.com/documentation/coreservices/1442744-uttypecopypreferredtagwithclass?language=objc
-
-    ext = CoreServices.UTTypeCopyPreferredTagWithClass(
-        uti, CoreServices.kUTTagClassFilenameExtension
-    )
-    return ext
+    with objc.autorelease_pool():
+        return CoreServices.UTTypeCopyPreferredTagWithClass(
+            uti, CoreServices.kUTTagClassFilenameExtension
+        )
 
 
 def findfiles(pattern, path_):
     """Returns list of filenames from path_ matched by pattern
-       shell pattern. Matching is case-insensitive."""
+       shell pattern. Matching is case-insensitive.
+       If 'path_' is invalid/doesn't exist, returns []."""
+    if not os.path.isdir(path_):
+        return []
     # See: https://gist.github.com/techtonik/5694830
 
     rule = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
@@ -365,117 +302,6 @@ def findfiles(pattern, path_):
 #     open_scpt.run()
 
 
-def _export_photo_uuid_applescript(
-    uuid,
-    dest,
-    filestem=None,
-    original=True,
-    edited=False,
-    live_photo=False,
-    timeout=120,
-    burst=False,
-):
-    """ Export photo to dest path using applescript to control Photos
-        If photo is a live photo, exports both the photo and associated .mov file
-        uuid: UUID of photo to export
-        dest: destination path to export to
-        filestem: (string) if provided, exported filename will be named stem.ext 
-                  where ext is extension of the file exported by photos (e.g. .jpeg, .mov, etc)
-                  If not provided, file will be named with whatever name Photos uses
-                  If filestem.ext exists, it wil be overwritten
-        original: (boolean) if True, export original image; default = True
-        edited: (boolean) if True, export edited photo; default = False
-                If photo not edited and edited=True, will still export the original image
-                caller must verify image has been edited
-        *Note*: must be called with either edited or original but not both, 
-                will raise error if called with both edited and original = True
-        live_photo: (boolean) if True, export associated .mov live photo; default = False
-        timeout: timeout value in seconds; export will fail if applescript run time exceeds timeout
-        burst: (boolean) set to True if file is a burst image to avoid Photos export error
-        Returns: list of paths to exported file(s) or None if export failed
-        Note: For Live Photos, if edited=True, will export a jpeg but not the movie, even if photo
-              has not been edited. This is due to how Photos Applescript interface works.
-    """
-
-    # setup the applescript to do the export
-    export_scpt = AppleScript(
-        """ 
-		on export_by_uuid(theUUID, thePath, original, edited, theTimeOut)
-			tell application "Photos"
-				set thePath to thePath
-				set theItem to media item id theUUID
-				set theFilename to filename of theItem
-				set itemList to {theItem}
-				
-				if original then
-					with timeout of theTimeOut seconds
-						export itemList to POSIX file thePath with using originals
-					end timeout
-				end if
-				
-				if edited then
-					with timeout of theTimeOut seconds
-						export itemList to POSIX file thePath
-					end timeout
-				end if
-				
-				return theFilename
-			end tell
-
-		end export_by_uuid
-		"""
-    )
-
-    dest = pathlib.Path(dest)
-    if not dest.is_dir:
-        raise ValueError(f"dest {dest} must be a directory")
-
-    if not original ^ edited:
-        raise ValueError(f"edited or original must be True but not both")
-
-    tmpdir = tempfile.TemporaryDirectory(prefix="osxphotos_")
-
-    # export original
-    filename = None
-    try:
-        filename = export_scpt.call(
-            "export_by_uuid", uuid, tmpdir.name, original, edited, timeout
-        )
-    except Exception as e:
-        logging.warning("Error exporting uuid %s: %s" % (uuid, str(e)))
-        return None
-
-    if filename is not None:
-        # need to find actual filename as sometimes Photos renames JPG to jpeg on export
-        # may be more than one file exported (e.g. if Live Photo, Photos exports both .jpeg and .mov)
-        # TemporaryDirectory will cleanup on return
-        filename_stem = pathlib.Path(filename).stem
-        files = glob.glob(os.path.join(tmpdir.name, "*"))
-        exported_paths = []
-        for fname in files:
-            path = pathlib.Path(fname)
-            if len(files) > 1 and not live_photo and path.suffix.lower() == ".mov":
-                # it's the .mov part of live photo but not requested, so don't export
-                logging.debug(f"Skipping live photo file {path}")
-                continue
-            if len(files) > 1 and burst and path.stem != filename_stem:
-                # skip any burst photo that's not the one we asked for
-                logging.debug(f"Skipping burst photo file {path}")
-                continue
-            if filestem:
-                # rename the file based on filestem, keeping original extension
-                dest_new = dest / f"{filestem}{path.suffix}"
-            else:
-                # use the name Photos provided
-                dest_new = dest / path.name
-            logging.debug(f"exporting {path} to dest_new: {dest_new}")
-            _copy_file(str(path), str(dest_new))
-            exported_paths.append(str(dest_new))
-        return exported_paths
-    else:
-        return None
-
-
 def _open_sql_file(dbname):
     """ opens sqlite file dbname in read-only mode
         returns tuple of (connection, cursor) """
@@ -527,7 +353,7 @@ def _db_is_locked(dbname):
 #     except KeyError:
 #         uuid_str = None
 #     return uuid_str
-    
+
 # def set_uuid_for_file(filepath, uuid):
 #     """ sets the UUID associated with an exported file
 #         filepath: path to exported photo
@@ -539,3 +365,39 @@ def _db_is_locked(dbname):
 #     attr = xattr.xattr(filepath)
 #     uuid_bytes = bytes(uuid, 'utf-8')
 #     attr.set(OSXPHOTOS_XATTR_UUID, uuid_bytes)
+
+
+def normalize_unicode(value):
+    """ normalize unicode data """
+    if value is None:
+        return None
+
+    if not isinstance(value, str):
+        raise ValueError("value must be str")
+    return unicodedata.normalize(UNICODE_FORMAT, value)
+
+
+def increment_filename(filepath):
+    """ Return filename (1).ext, etc if filename.ext exists
+
+        If file exists in filename's parent folder with same stem as filename, 
+        add (1), (2), etc. until a non-existing filename is found.
+
+    Args:
+        filepath: str; full path, including file name
+
+    Returns:
+        new filepath (or same if not incremented)
+
+    Note: This obviously is subject to race condition so using with caution.
+    """
+    dest = pathlib.Path(str(filepath))
+    count = 1
+    dest_files = findfiles(f"{dest.stem}*", str(dest.parent))
+    dest_files = [pathlib.Path(f).stem.lower() for f in dest_files]
+    dest_new = dest.stem
+    while dest_new.lower() in dest_files:
+        dest_new = f"{dest.stem} ({count})"
+        count += 1
+    dest = dest.parent / f"{dest_new}{dest.suffix}"
+    return str(dest)

requirements.txt

@@ -1,19 +1,38 @@
+aiohttp==4.0.0a1
 altgraph==0.17
 ansimarkup==1.4.0
 appdirs==1.4.3
+appnope==0.1.0
 astroid==2.2.5
+async-timeout==3.0.1
 atomicwrites==1.3.0
 attrs==19.1.0
+backcall==0.1.0
 better-exceptions-fork==0.2.1.post6
-# bpylist2==2.0.3;python_version<"3.8"
-https://github.com/RhetTbull/bpylist/releases/download/v2.0.3/bpylist2-2.0.3.tar.gz#egg=bpylist2;python_version<"3.8"
-bpylist2==3.0.0;python_version>="3.8"
-certifi==2019.3.9
+black==19.10b0
+bleach==3.1.4
+bpylist2==3.0.2
+certifi==2020.4.5.1
+cffi==1.14.0
+chardet==3.0.4
 Click==7.0
 colorama==0.4.1
 coverage==4.5.4
-importlib-metadata>=0.18
+decorator==4.4.2
+distlib==0.3.1
+docutils==0.16
+entrypoints==0.3
+filelock==3.0.12
+idna==2.9
+importlib-metadata==1.6.0
+ipykernel==5.1.4
+ipython==7.13.0
+ipython-genutils==0.2.0
 isort==4.3.20
+jedi==0.16.0
+jupyter-client==6.1.2
+jupyter-core==4.6.3
+keyring==21.2.0
 lazy-object-proxy==1.4.1
 loguru==0.2.5
 macholib==1.14
@@ -22,137 +41,169 @@ MarkupSafe==1.1.1
 mccabe==0.6.1
 modulegraph==0.18
 more-itertools==7.2.0
+multidict==4.7.6
+osxmetadata>=0.99.11
 packaging==19.0
+parso==0.6.2
 pathspec==0.7.0
 pathvalidate==2.2.1
+pexpect==4.8.0
+photoscript==0.1.0
+pickleshare==0.7.5
+Pillow==7.2.0
+pkginfo==1.5.0.1
 pluggy==0.12.0
+prompt-toolkit==3.0.4
+psutil==5.7.0
+ptyprocess==0.6.0
 py==1.8.0
 py2app==0.21
-Pygments==2.4.2
+pycparser==2.20
+pyfiglet==0.8.post1
+Pygments==2.6.1
 PyInstaller==3.6
 pyinstaller-setuptools==2019.3
 pylint==2.3.1
-pyobjc==6.0.1
-pyobjc-core==6.0.1
-pyobjc-framework-Accounts==6.0.1
-pyobjc-framework-AddressBook==6.0.1
-pyobjc-framework-AdSupport==6.0.1
-pyobjc-framework-AppleScriptKit==6.0.1
-pyobjc-framework-AppleScriptObjC==6.0.1
-pyobjc-framework-ApplicationServices==6.0.1
-pyobjc-framework-AuthenticationServices==6.0.1
-pyobjc-framework-Automator==6.0.1
-pyobjc-framework-AVFoundation==6.0.1
-pyobjc-framework-AVKit==6.0.1
-pyobjc-framework-BusinessChat==6.0.1
-pyobjc-framework-CalendarStore==6.0.1
-pyobjc-framework-CFNetwork==6.0.1
-pyobjc-framework-CloudKit==6.0.1
-pyobjc-framework-Cocoa==6.0.1
-pyobjc-framework-Collaboration==6.0.1
-pyobjc-framework-ColorSync==6.0.1
-pyobjc-framework-Contacts==6.0.1
-pyobjc-framework-ContactsUI==6.0.1
-pyobjc-framework-CoreAudio==6.0.1
-pyobjc-framework-CoreAudioKit==6.0.1
-pyobjc-framework-CoreBluetooth==6.0.1
-pyobjc-framework-CoreData==6.0.1
-pyobjc-framework-CoreHaptics==6.0.1
-pyobjc-framework-CoreLocation==6.0.1
-pyobjc-framework-CoreMedia==6.0.1
-pyobjc-framework-CoreMediaIO==6.0.1
-pyobjc-framework-CoreML==6.0.1
-pyobjc-framework-CoreMotion==6.0.1
-pyobjc-framework-CoreServices==6.0.1
-pyobjc-framework-CoreSpotlight==6.0.1
-pyobjc-framework-CoreText==6.0.1
-pyobjc-framework-CoreWLAN==6.0.1
-pyobjc-framework-CryptoTokenKit==6.0.1
-pyobjc-framework-DeviceCheck==6.0.1
-pyobjc-framework-DictionaryServices==6.0.1
-pyobjc-framework-DiscRecording==6.0.1
-pyobjc-framework-DiscRecordingUI==6.0.1
-pyobjc-framework-DiskArbitration==6.0.1
-pyobjc-framework-DVDPlayback==6.0.1
-pyobjc-framework-EventKit==6.0.1
-pyobjc-framework-ExceptionHandling==6.0.1
-pyobjc-framework-ExecutionPolicy==6.0.1
-pyobjc-framework-ExternalAccessory==6.0.1
-pyobjc-framework-FileProvider==6.0.1
-pyobjc-framework-FileProviderUI==6.0.1
-pyobjc-framework-FinderSync==6.0.1
-pyobjc-framework-FSEvents==6.0.1
-pyobjc-framework-GameCenter==6.0.1
-pyobjc-framework-GameController==6.0.1
-pyobjc-framework-GameKit==6.0.1
-pyobjc-framework-GameplayKit==6.0.1
-pyobjc-framework-ImageCaptureCore==6.0.1
-pyobjc-framework-IMServicePlugIn==6.0.1
-pyobjc-framework-InputMethodKit==6.0.1
-pyobjc-framework-InstallerPlugins==6.0.1
-pyobjc-framework-InstantMessage==6.0.1
-pyobjc-framework-Intents==6.0.1
-pyobjc-framework-IOSurface==6.0.1
-pyobjc-framework-iTunesLibrary==6.0.1
-pyobjc-framework-LatentSemanticMapping==6.0.1
-pyobjc-framework-LaunchServices==6.0.1
-pyobjc-framework-libdispatch==6.0.1
-pyobjc-framework-LinkPresentation==6.0.1
-pyobjc-framework-LocalAuthentication==6.0.1
-pyobjc-framework-MapKit==6.0.1
-pyobjc-framework-MediaAccessibility==6.0.1
-pyobjc-framework-MediaLibrary==6.0.1
-pyobjc-framework-MediaPlayer==6.0.1
-pyobjc-framework-MediaToolbox==6.0.1
-pyobjc-framework-MetalKit==6.0.1
-pyobjc-framework-ModelIO==6.0.1
-pyobjc-framework-MultipeerConnectivity==6.0.1
-pyobjc-framework-NaturalLanguage==6.0.1
-pyobjc-framework-NetFS==6.0.1
-pyobjc-framework-Network==6.0.1
-pyobjc-framework-NetworkExtension==6.0.1
-pyobjc-framework-NotificationCenter==6.0.1
-pyobjc-framework-OpenDirectory==6.0.1
-pyobjc-framework-OSAKit==6.0.1
-pyobjc-framework-OSLog==6.0.1
-pyobjc-framework-PencilKit==6.0.1
-pyobjc-framework-Photos==6.0.1
-pyobjc-framework-PhotosUI==6.0.1
-pyobjc-framework-PreferencePanes==6.0.1
-pyobjc-framework-PubSub==6.0.1
-pyobjc-framework-PushKit==6.0.1
+pyobjc==6.2.2
+pyobjc-core==6.2.2
+pyobjc-framework-Accounts==6.2.2
+pyobjc-framework-AddressBook==6.2.2
+pyobjc-framework-AdSupport==6.2.2
+pyobjc-framework-AppleScriptKit==6.2.2
+pyobjc-framework-AppleScriptObjC==6.2.2
+pyobjc-framework-ApplicationServices==6.2.2
+pyobjc-framework-AuthenticationServices==6.2.2
+pyobjc-framework-AutomaticAssessmentConfiguration==6.2.2
+pyobjc-framework-Automator==6.2.2
+pyobjc-framework-AVFoundation==6.2.2
+pyobjc-framework-AVKit==6.2.2
+pyobjc-framework-BusinessChat==6.2.2
+pyobjc-framework-CalendarStore==6.2.2
+pyobjc-framework-CFNetwork==6.2.2
+pyobjc-framework-CloudKit==6.2.2
+pyobjc-framework-Cocoa==6.2.2
+pyobjc-framework-Collaboration==6.2.2
+pyobjc-framework-ColorSync==6.2.2
+pyobjc-framework-Contacts==6.2.2
+pyobjc-framework-ContactsUI==6.2.2
+pyobjc-framework-CoreAudio==6.2.2
+pyobjc-framework-CoreAudioKit==6.2.2
+pyobjc-framework-CoreBluetooth==6.2.2
+pyobjc-framework-CoreData==6.2.2
+pyobjc-framework-CoreHaptics==6.2.2
+pyobjc-framework-CoreLocation==6.2.2
+pyobjc-framework-CoreMedia==6.2.2
+pyobjc-framework-CoreMediaIO==6.2.2
+pyobjc-framework-CoreML==6.2.2
+pyobjc-framework-CoreMotion==6.2.2
+pyobjc-framework-CoreServices==6.2.2
+pyobjc-framework-CoreSpotlight==6.2.2
+pyobjc-framework-CoreText==6.2.2
+pyobjc-framework-CoreWLAN==6.2.2
+pyobjc-framework-CryptoTokenKit==6.2.2
+pyobjc-framework-DeviceCheck==6.2.2
+pyobjc-framework-DictionaryServices==6.2.2
+pyobjc-framework-DiscRecording==6.2.2
+pyobjc-framework-DiscRecordingUI==6.2.2
+pyobjc-framework-DiskArbitration==6.2.2
+pyobjc-framework-DVDPlayback==6.2.2
+pyobjc-framework-EventKit==6.2.2
+pyobjc-framework-ExceptionHandling==6.2.2
+pyobjc-framework-ExecutionPolicy==6.2.2
+pyobjc-framework-ExternalAccessory==6.2.2
+pyobjc-framework-FileProvider==6.2.2
+pyobjc-framework-FileProviderUI==6.2.2
+pyobjc-framework-FinderSync==6.2.2
+pyobjc-framework-FSEvents==6.2.2
+pyobjc-framework-GameCenter==6.2.2
+pyobjc-framework-GameController==6.2.2
+pyobjc-framework-GameKit==6.2.2
+pyobjc-framework-GameplayKit==6.2.2
+pyobjc-framework-ImageCaptureCore==6.2.2
+pyobjc-framework-IMServicePlugIn==6.2.2
+pyobjc-framework-InputMethodKit==6.2.2
+pyobjc-framework-InstallerPlugins==6.2.2
+pyobjc-framework-InstantMessage==6.2.2
+pyobjc-framework-Intents==6.2.2
+pyobjc-framework-IOSurface==6.2.2
+pyobjc-framework-iTunesLibrary==6.2.2
+pyobjc-framework-LatentSemanticMapping==6.2.2
+pyobjc-framework-LaunchServices==6.2.2
+pyobjc-framework-libdispatch==6.2.2
+pyobjc-framework-LinkPresentation==6.2.2
+pyobjc-framework-LocalAuthentication==6.2.2
+pyobjc-framework-MapKit==6.2.2
+pyobjc-framework-MediaAccessibility==6.2.2
+pyobjc-framework-MediaLibrary==6.2.2
+pyobjc-framework-MediaPlayer==6.2.2
+pyobjc-framework-MediaToolbox==6.2.2
+pyobjc-framework-Metal==6.2.2
+pyobjc-framework-MetalKit==6.2.2
+pyobjc-framework-ModelIO==6.2.2
+pyobjc-framework-MultipeerConnectivity==6.2.2
+pyobjc-framework-NaturalLanguage==6.2.2
+pyobjc-framework-NetFS==6.2.2
+pyobjc-framework-Network==6.2.2
+pyobjc-framework-NetworkExtension==6.2.2
+pyobjc-framework-NotificationCenter==6.2.2
+pyobjc-framework-OpenDirectory==6.2.2
+pyobjc-framework-OSAKit==6.2.2
+pyobjc-framework-OSLog==6.2.2
+pyobjc-framework-PencilKit==6.2.2
+pyobjc-framework-Photos==6.2.2
+pyobjc-framework-PhotosUI==6.2.2
+pyobjc-framework-PreferencePanes==6.2.2
+pyobjc-framework-PubSub==6.2
+pyobjc-framework-PushKit==6.2.2
 pyobjc-framework-QTKit==6.0.1
-pyobjc-framework-Quartz==6.0.1
-pyobjc-framework-QuickLookThumbnailing==6.0.1
-pyobjc-framework-SafariServices==6.0.1
-pyobjc-framework-SceneKit==6.0.1
-pyobjc-framework-ScreenSaver==6.0.1
-pyobjc-framework-ScriptingBridge==6.0.1
-pyobjc-framework-SearchKit==6.0.1
-pyobjc-framework-Security==6.0.1
-pyobjc-framework-SecurityFoundation==6.0.1
-pyobjc-framework-SecurityInterface==6.0.1
-pyobjc-framework-ServiceManagement==6.0.1
-pyobjc-framework-Social==6.0.1
-pyobjc-framework-SoundAnalysis==6.0.1
-pyobjc-framework-Speech==6.0.1
-pyobjc-framework-SpriteKit==6.0.1
-pyobjc-framework-StoreKit==6.0.1
-pyobjc-framework-SyncServices==6.0.1
-pyobjc-framework-SystemConfiguration==6.0.1
-pyobjc-framework-SystemExtensions==6.0.1
-pyobjc-framework-UserNotifications==6.0.1
-pyobjc-framework-VideoSubscriberAccount==6.0.1
-pyobjc-framework-VideoToolbox==6.0.1
-pyobjc-framework-Vision==6.0.1
-pyobjc-framework-WebKit==6.0.1
+pyobjc-framework-Quartz==6.2.2
+pyobjc-framework-QuickLookThumbnailing==6.2.2
+pyobjc-framework-SafariServices==6.2.2
+pyobjc-framework-SceneKit==6.2.2
+pyobjc-framework-ScreenSaver==6.2.2
+pyobjc-framework-ScriptingBridge==6.2.2
+pyobjc-framework-SearchKit==6.2.2
+pyobjc-framework-Security==6.2.2
+pyobjc-framework-SecurityFoundation==6.2.2
+pyobjc-framework-SecurityInterface==6.2.2
+pyobjc-framework-ServiceManagement==6.2.2
+pyobjc-framework-Social==6.2.2
+pyobjc-framework-SoundAnalysis==6.2.2
+pyobjc-framework-Speech==6.2.2
+pyobjc-framework-SpriteKit==6.2.2
+pyobjc-framework-StoreKit==6.2.2
+pyobjc-framework-SyncServices==6.2.2
+pyobjc-framework-SystemConfiguration==6.2.2
+pyobjc-framework-SystemExtensions==6.2.2
+pyobjc-framework-UserNotifications==6.2.2
+pyobjc-framework-VideoSubscriberAccount==6.2.2
+pyobjc-framework-VideoToolbox==6.2.2
+pyobjc-framework-Vision==6.2.2
+pyobjc-framework-WebKit==6.2.2
 pyparsing==2.4.1.1
+python-dateutil==2.8.1
 PyYAML==5.1.2
+pyzmq==18.1.1
+readme-renderer==25.0
 regex==2020.2.20
-six==1.12.0
+requests==2.23.0
+requests-toolbelt==0.9.1
+six==1.14.0
 termcolor==1.1.0
 toml==0.10.0
+tornado==6.0.4
+tox==3.19.0
+tox-conda==0.2.1
+tqdm==4.45.0
+traitlets==4.3.3
+twine==3.1.1
 typed-ast==1.4.1
-wcwidth==0.1.7
+typing-extensions==3.7.4.2
+urllib3==1.25.9
+virtualenv==20.0.30
+wcwidth==0.1.9
+webencodings==0.5.1
 wrapt==1.11.1
-zipp==0.5.2
+wurlitzer==2.0.1
+yarl==1.4.2
+zipp==0.5.2

setup.py

@@ -3,7 +3,7 @@
 #
 # setup.py script for osxphotos
 #
-# Copyright (c) 2019, 2020 Rhet Turnbull, rturnbull+git@gmail.com
+# Copyright (c) 2019, 2020, 2021 Rhet Turnbull, rturnbull+git@gmail.com
 # All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person
@@ -45,26 +45,15 @@ with open(
     exec(f.read(), about)
 
 # read README.md into long_description
-with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
+with open(os.path.join(this_directory, "README.rst"), encoding="utf-8") as f:
     about["long_description"] = f.read()
 
-# ugly hack to install custom version of bpylist2 needed for Python < 3.8
-# the stock version of bylist2==2.0.3 causes an error related to
-# "pkg_resources.ContextualVersionConflict: (pycodestyle 2.3.1..."
-# PEP 508 no help here as URL-based lookups not allowed in PyPI packages
-# if you know a better way, PRs welcome!
-# once I go to 3.8+ required, this won't be necessary as bpylist2 3.0+ solves this issue
-if py_ver < 3.8:
-    os.system(
-        "python3 -m pip install git+git://github.com/RhetTbull/bpylist2.git#egg=bpylist2"
-    )
-
 setup(
     name="osxphotos",
     version=about["__version__"],
-    description="Manipulate (read-only) Apple's Photos app library on Mac OS X",
+    description="Export photos from Apple's macOS Photos app and query the Photos library database to access metadata about images.",
     long_description=about["long_description"],
-    long_description_content_type="text/markdown",
+    long_description_content_type="text/x-rst",
     author="Rhet Turnbull",
     author_email="rturnbull+git@gmail.com",
     url="https://github.com/RhetTbull/",
@@ -78,18 +67,23 @@ setup(
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Operating System :: MacOS :: MacOS X",
+        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
     install_requires=[
-        "pyobjc>=6.0.1",
+        "pyobjc>=6.2.2",
         "Click>=7",
         "PyYAML>=5.1.2",
         "Mako>=1.1.1",
-        "bpylist2==2.0.3;python_version<'3.8'",
-        "bpylist2==3.0.0;python_version>='3.8'",
+        "bpylist2==3.0.2",
         "pathvalidate==2.2.1",
         "dataclasses==0.7;python_version<'3.7'",
+        "wurlitzer>=2.0.1",
+        "photoscript>=0.1.0",
+        "toml>=0.10.0",
+        "osxmetadata>=0.99.13",
     ],
     entry_points={"console_scripts": ["osxphotos=osxphotos.__main__:cli"]},
     include_package_data=True,

tests/README.md

@@ -17,14 +17,29 @@ Some of the export tests rely on photos in my local library and will look for `O
 
 One test for locale does not run on GitHub's automated workflow and will look for `OSXPHOTOS_TEST_LOCALE=1` to determine if it should be run.  If you want to run this test, set the environment variable.  
 
-## Attribution ##
-These tests utilize a test Photos library. The test library is populated with photos from [flickr](https://www.flickr.com).  All images used are licensed under Creative Commons 2.0 Attribution [license](https://creativecommons.org/licenses/by/2.0/).  
+## Test Photo Libraries
+**Important**: The test code uses several test photo libraries created on various version of MacOS.  If you need to inspect one of these or modify one for a test, make a copy of the library (for example, copy it to your ~/Pictures folder) then open the copy in Photos.  Once done, copy the revised library back to the tests/ folder.  If you do not do this, the Photos background process photoanalysisd will forever try to process the library resulting in updates to the database which will cause git to see changes to the file you didn't intend.  I'm not aware of any way to disassociate photoanalysisd from the library once you've opened it in Photos.
 
-Images used from:
+## Attribution ##
+These tests utilize a test Photos library. The test library is populated with photos from [flickr](https://www.flickr.com) and from my own photo library.  All images used are licensed under Creative Commons 2.0 Attribution [license](https://creativecommons.org/licenses/by/2.0/).  
+
+Flickr images used from:
 - [Jeff Hitchcock](https://www.flickr.com/photos/arbron/48353451872/)
 - [Carlos Montesdeoca](https://www.flickr.com/photos/carlosmontesdeocastudio)
 - [Rydale Clothing](https://www.flickr.com/photos/rydaleclothing)
 - [Marco Verch](https://www.flickr.com/photos/30478819@N08/48228222317/)
 - [K M](https://www.flickr.com/photos/153387643@N08/49334338022/)
-
+- [Shelby Mash](https://www.flickr.com/photos/shelbzyleigh/3809603052)
+- [Rory MacLeod](https://www.flickr.com/photos/macrj/6969547134)
+- [Md. Al Amin](https://www.flickr.com/photos/alamin_bd/45207044465)
+- [Fatlum Haliti](https://www.flickr.com/photos/lumlumi/363449752)
+- [Benny Mazur](https://www.flickr.com/photos/benimoto/399012465)
+- [Sara Cooper PR](https://www.flickr.com/photos/saracooperpr/6422472677)
+- [herval](https://www.flickr.com/photos/herval/2403994289)
+- [Vox Efx](https://www.flickr.com/photos/vox_efx/141137669)
+- [Bill Strain](https://www.flickr.com/photos/billstrain/5117042252)
+- [Guilherme Yagui](https://www.flickr.com/photos/yagui7/15895161088/)
+- [Deborah Austin](https://www.flickr.com/photos/littledebbie11/8703591799/)
+- [We Are Social](https://www.flickr.com/photos/wearesocial/23309711462/)
+- [cloud.shepherd](https://www.flickr.com/photos/exnucboy1/31017877125)
 

tests/Test-10.14.6.photoslibrary/private/com.apple.Photos/appPrivateData.plist

@@ -36,7 +36,7 @@
 			<key>other</key>
 			<integer>0</integer>
 			<key>photos</key>
-			<integer>6</integer>
+			<integer>11</integer>
 			<key>videos</key>
 			<integer>0</integer>
 		</dict>

tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist

@@ -3,8 +3,8 @@
 <plist version="1.0">
 <dict>
 	<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
-	<date>2020-04-25T23:54:43Z</date>
+	<date>2020-11-01T02:34:49Z</date>
 	<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
-	<date>2020-04-26T06:26:10Z</date>
+	<date>2020-11-01T02:34:49Z</date>
 </dict>
 </plist>

tests/Test-10.14.6.photoslibrary/private/com.apple.photomodel/appPrivateData.plist

@@ -5,7 +5,7 @@
 	<key>LithiumMessageTracer</key>
 	<dict>
 		<key>LastReportedDate</key>
-		<date>2020-04-17T17:51:16Z</date>
+		<date>2020-10-04T23:49:39Z</date>
 	</dict>
 </dict>
 </plist>

tests/Test-10.14.6.photoslibrary/resources/moments/analysismetadata.plist

@@ -11,6 +11,6 @@
 	<key>PLLastRevGeoForcedProviderOutOfDateCheckVersionKey</key>
 	<integer>1</integer>
 	<key>PLLastRevGeoVerFileFetchDateKey</key>
-	<date>2020-04-25T23:54:29Z</date>
+	<date>2020-11-01T02:34:46Z</date>
 </dict>
 </plist>

tests/Test-10.14.6.photoslibrary/resources/moments/historicalmarker.plist

@@ -3,7 +3,7 @@
 <plist version="1.0">
 <dict>
 	<key>LastHistoryRowId</key>
-	<integer>606</integer>
+	<integer>948</integer>
 	<key>LibraryBuildTag</key>
 	<string>D8C4AAA1-3AB6-4A65-BEBD-99CC3E5D433E</string>
 	<key>LibrarySchemaVersion</key>