Fix for #575, database version 5001

* docs: update README.md [skip ci]

* docs: update .all-contributorsrc [skip ci]

Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>

.all-contributorsrc

@@ -293,7 +293,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/6291?v=4",
       "profile": "https://hyfen.net",
       "contributions": [
-        "doc", "code"
+        "doc",
+        "code"
       ]
     },
     {
@@ -304,6 +305,16 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "ahti123",
+      "name": "Ahti Liin",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22232632?v=4",
+      "profile": "https://github.com/ahti123",
+      "contributions": [
+        "code",
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

CHANGELOG.md

@@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.44.7](https://github.com/RhetTbull/osxphotos/compare/v0.44.6...v0.44.7)
+
+> 8 January 2022
+
+- Fix for #576, error exporting edited live photos [`2e7db47`](https://github.com/RhetTbull/osxphotos/commit/2e7db47806683fdd0db4d1d75e42471d2f127d4d)
+
+#### [v0.44.6](https://github.com/RhetTbull/osxphotos/compare/v0.44.5...v0.44.6)
+
+> 6 January 2022
+
+- Fix for burst images with pick type = 0, partial fix for #571 [`d2d56a7`](https://github.com/RhetTbull/osxphotos/commit/d2d56a7f7118aeffa7ac81cc474fdd4fb4843065)
+
+#### [v0.44.5](https://github.com/RhetTbull/osxphotos/compare/v0.44.4...v0.44.5)
+
+> 6 January 2022
+
+- More refactoring of export code, #462 [`0c9bd87`](https://github.com/RhetTbull/osxphotos/commit/0c9bd8760261770e11b0fa59153f49f2d65e2c2f)
+- Fix for #570 [`661a573`](https://github.com/RhetTbull/osxphotos/commit/661a573bf50353fb2393c604080ffe0790ade59c)
+- version bump [skip ci] [`b4897ff`](https://github.com/RhetTbull/osxphotos/commit/b4897ff1b5d2bc00f34158345b2b5fe85f1490ac)
+
 #### [v0.44.4](https://github.com/RhetTbull/osxphotos/compare/v0.44.3...v0.44.4)
 
 > 4 January 2022

README.md

@@ -5,7 +5,7 @@
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/osxphotos)
 [![Downloads](https://static.pepy.tech/personalized-badge/osxphotos?period=month&units=international_system&left_color=black&right_color=brightgreen&left_text=downloads/month)](https://pepy.tech/project/osxphotos)
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-32-orange.svg?style=flat)](#contributors)
+[![All Contributors](https://img.shields.io/badge/all_contributors-33-orange.svg?style=flat)](#contributors)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 OSXPhotos provides the ability to interact with and query Apple's Photos.app library on macOS. You can query the Photos library database — for example, file name, file path, and metadata such as keywords/tags, persons/faces, albums, etc. You can also easily export both the original and edited photos.
@@ -1720,7 +1720,7 @@ Substitution                    Description
 {lf}                            A line feed: '\n', alias for {newline}
 {cr}                            A carriage return: '\r'
 {crlf}                          a carriage return + line feed: '\r\n'
-{osxphotos_version}             The osxphotos version, e.g. '0.44.6'
+{osxphotos_version}             The osxphotos version, e.g. '0.44.8'
 {osxphotos_cmd_line}            The full command line used to run osxphotos
 
 The following substitutions may result in multiple values. Thus if specified for
@@ -3622,7 +3622,7 @@ The following template field substitutions are availabe for use the templating s
 |{lf}|A line feed: '\n', alias for {newline}|
 |{cr}|A carriage return: '\r'|
 |{crlf}|a carriage return + line feed: '\r\n'|
-|{osxphotos_version}|The osxphotos version, e.g. '0.44.6'|
+|{osxphotos_version}|The osxphotos version, e.g. '0.44.8'|
 |{osxphotos_cmd_line}|The full command line used to run osxphotos|
 |{album}|Album(s) photo is contained in|
 |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|
@@ -3850,6 +3850,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://alandefreitas.github.io/alandefreitas/"><img src="https://avatars.githubusercontent.com/u/5369819?v=4?s=75" width="75px;" alt=""/><br /><sub><b>Alan de Freitas</b></sub></a><br /><a href="https://github.com/RhetTbull/osxphotos/issues?q=author%3Aalandefreitas" title="Bug reports">🐛</a></td>
     <td align="center"><a href="https://hyfen.net"><img src="https://avatars.githubusercontent.com/u/6291?v=4?s=75" width="75px;" alt=""/><br /><sub><b>Andrew Louis</b></sub></a><br /><a href="https://github.com/RhetTbull/osxphotos/commits?author=hyfen" title="Documentation">📖</a> <a href="https://github.com/RhetTbull/osxphotos/commits?author=hyfen" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/neebah"><img src="https://avatars.githubusercontent.com/u/71442026?v=4?s=75" width="75px;" alt=""/><br /><sub><b>neebah</b></sub></a><br /><a href="https://github.com/RhetTbull/osxphotos/issues?q=author%3Aneebah" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/ahti123"><img src="https://avatars.githubusercontent.com/u/22232632?v=4?s=75" width="75px;" alt=""/><br /><sub><b>Ahti Liin</b></sub></a><br /><a href="https://github.com/RhetTbull/osxphotos/commits?author=ahti123" title="Code">💻</a> <a href="https://github.com/RhetTbull/osxphotos/issues?q=author%3Aahti123" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 12e2b2711a035185a2f8b8e500263a8d
+config: fff79f4920939baa44eddc90423972ec
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '0.44.6',
+    VERSION: '0.44.8',
     LANGUAGE: 'None',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

docs/cli.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.6 documentation</title>
+    <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/genindex.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Index &#8212; osxphotos 0.44.6 documentation</title>
+    <title>Index &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/index.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.6 documentation</title>
+    <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/modules.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos &#8212; osxphotos 0.44.6 documentation</title>
+    <title>osxphotos &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/reference.html

@@ -6,7 +6,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
-    <title>osxphotos package &#8212; osxphotos 0.44.6 documentation</title>
+    <title>osxphotos package &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

docs/search.html

@@ -5,7 +5,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Search &#8212; osxphotos 0.44.6 documentation</title>
+    <title>Search &#8212; osxphotos 0.44.8 documentation</title>
     <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
     <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
     

osxphotos/_constants.py

@@ -20,8 +20,8 @@ UNICODE_FORMAT = "NFC"
 # Photos 3.0 (10.13.6) == 3301
 # Photos 4.0 (10.14.5) == 4016
 # Photos 4.0 (10.14.6) == 4025
-# Photos 5.0 (10.15.0) == 6000
+# Photos 5.0 (10.15.0) == 6000 or 5001
-_TESTED_DB_VERSIONS = ["6000", "4025", "4016", "3301", "2622"]
+_TESTED_DB_VERSIONS = ["6000", "5001", "4025", "4016", "3301", "2622"]
 
 # database model versions (applies to Photos 5, Photos 6)
 # these come from PLModelVersion key in binary plist in Z_METADATA.Z_PLIST
@@ -37,7 +37,7 @@ _PHOTOS_3_VERSION = "3301"
 
 # versions 5.0 and later have a different database structure
 _PHOTOS_4_VERSION = "4025"  # latest Mojove version on 10.14.6
-_PHOTOS_5_VERSION = "6000"  # seems to be current on 10.15.1 through 10.15.7 (also Big Sur and Monterey which switch to model version)
+_PHOTOS_5_VERSION = "5000"  # I've seen both 5001 and 6000.  6000 is most common on Catalina and up but there are some version 5001 database in the wild
 
 # Ranges for model version by Photos version
 _PHOTOS_5_MODEL_VERSION = [13000, 13999]

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.44.7"
+__version__ = "0.44.8"

osxphotos/cli.py

@@ -20,7 +20,7 @@ import osxmetadata
 import photoscript
 import rich.traceback
 import yaml
-from rich import pretty
+from rich import pretty, print
 
 import osxphotos
 
@@ -60,9 +60,11 @@ from .photoexporter import ExportResults, PhotoExporter
 from .photoinfo import PhotoInfo
 from .photokit import check_photokit_authorization, request_photokit_authorization
 from .photosalbum import PhotosAlbum
+from .photosdb.photosdb_utils import get_photos_library_version
 from .phototemplate import PhotoTemplate, RenderOptions
 from .pyrepl import embed_repl
 from .queryoptions import QueryOptions
+from .sqlgrep import sqlgrep
 from .uti import get_preferred_uti_extension
 from .utils import expand_and_validate_filepath, load_function, normalize_fs_path
 
@@ -4283,3 +4285,49 @@ def repl(ctx, cli_obj, db, emacs):
         quit_words=["q", "quit", "exit"],
         vi_mode=not emacs,
     )
+
+
+@cli.command(hidden=True)
+@DB_OPTION
+@click.pass_obj
+@click.pass_context
+@click.option(
+    "--ignore-case",
+    "-i",
+    required=False,
+    is_flag=True,
+    default=False,
+    help="Ignore case when searching (default is case-sensitive)",
+)
+@click.option(
+    "--print-filename",
+    "-p",
+    required=False,
+    is_flag=True,
+    default=False,
+    help="Print name of database file when printing results",
+)
+@click.argument("pattern", metavar="PATTERN", required=True)
+def grep(ctx, cli_obj, db, ignore_case, print_filename, pattern):
+    """Search for PATTERN in the Photos sqlite database file"""
+    db = db or get_photos_db()
+    db = pathlib.Path(db)
+    if db.is_file():
+        # if passed the actual database, really want the parent of the database directory
+        db = db.parent.parent
+    photos_ver = get_photos_library_version(str(db))
+    if photos_ver < 5:
+        db_file = db / "database" / "photos.db"
+    else:
+        db_file = db / "database" / "Photos.sqlite"
+
+    if not db_file.is_file():
+        click.secho(f"Could not find database file {db_file}", fg="red")
+        ctx.exit(2)
+
+    db_file = str(db_file)
+
+    for table, column, row_id, value in sqlgrep(
+        db_file, pattern, ignore_case, print_filename, rich_markup=True
+    ):
+        print(", ".join([table, column, row_id, value]))

osxphotos/photosdb/photosdb_utils.py

@@ -1,6 +1,7 @@
 """ utility functions used by PhotosDB """
 
 import logging
+import pathlib
 import plistlib
 
 from .._constants import (
@@ -17,7 +18,7 @@ from ..utils import _open_sql_file
 
 
 def get_db_version(db_file):
-    """ Gets the Photos DB version from LiGlobals table
+    """Gets the Photos DB version from LiGlobals table
 
     Args:
         db_file: path to photos.db database file containing LiGlobals table
@@ -44,7 +45,7 @@ def get_db_version(db_file):
 
 
 def get_model_version(db_file):
-    """ Returns the database model version from Z_METADATA
+    """Returns the database model version from Z_METADATA
 
     Args:
         db_file: path to Photos.sqlite database file containing Z_METADATA table
@@ -67,7 +68,7 @@ def get_model_version(db_file):
 
 
 def get_db_model_version(db_file):
-    """ Returns Photos version based on model version found in db_file
+    """Returns Photos version based on model version found in db_file
 
     Args:
         db_file: path to Photos.sqlite file
@@ -94,7 +95,7 @@ class UnknownLibraryVersion(Exception):
 
 
 def get_photos_library_version(library_path):
-    """Return int indicating which Photos version a library was created with """
+    """Return int indicating which Photos version a library was created with"""
     library_path = pathlib.Path(library_path)
     db_ver = get_db_version(str(library_path / "database" / "photos.db"))
     db_ver = int(db_ver)

osxphotos/sqlgrep.py

@@ -0,0 +1,55 @@
+"""Search through a sqlite database file for a given string"""
+
+import re
+import sqlite3
+from typing import Generator, List
+
+
+def sqlgrep(
+    filename: str,
+    pattern: str,
+    ignore_case: bool = False,
+    print_filename: bool = True,
+    rich_markup: bool = False,
+) -> Generator[List[str], None, None]:
+    """grep through a sqlite database file for a given string
+
+    Args:
+        filename (str): The filename of the sqlite database file
+        pattern (str): The pattern to search for
+        ignore_case (bool, optional): Ignore case when searching. Defaults to False.
+        print_filename (bool, optional): include the filename of the file with table name. Defaults to True.
+        rich_markup (bool, optional): Add rich markup to mark found text in bold. Defaults to False.
+
+    Returns:
+        Generator which yields list of [table, column, row_id, value]
+    """
+    flags = re.IGNORECASE if ignore_case else 0
+    try:
+        with sqlite3.connect(f"file:{filename}?mode=ro", uri=True) as conn:
+            regex = re.compile(r"(" + pattern + r")", flags=flags)
+            filename_header = f"{filename}: " if print_filename else ""
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+            for tablerow in cursor.fetchall():
+                table = tablerow[0]
+                cursor.execute("SELECT * FROM {t}".format(t=table))
+                for row_num, row in enumerate(cursor):
+                    for field in row.keys():
+                        field_value = row[field]
+                        if not field_value or type(field_value) == bytes:
+                            # don't search binary blobs
+                            next
+                        field_value = str(field_value)
+                        if re.search(pattern, field_value, flags=flags):
+                            if rich_markup:
+                                field_value = regex.sub(r"[bold]\1[/bold]", field_value)
+                            yield [
+                                f"{filename_header}{table}",
+                                field,
+                                str(row_num),
+                                field_value,
+                            ]
+    except sqlite3.DatabaseError as e:
+        raise sqlite3.DatabaseError(f"{filename}: {e}")

tests/test_cli.py

@@ -768,7 +768,10 @@ CLI_EXPORT_UUID_FROM_FILE_FILENAMES = [
     "wedding_edited.jpeg",
 ]
 
-CLI_EXPORT_SKIP_UUID = ["E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51", "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4"]
+CLI_EXPORT_SKIP_UUID = [
+    "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51",
+    "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
+]
 CLI_EXPORT_SKIP_UUID_FILENAMES = [
     "Tulips.jpg",
     "Tulips_edited.jpeg",
@@ -903,6 +906,14 @@ QUERY_EXIF_DATA_CASE_INSENSITIVE = [
 ]
 EXPORT_EXIF_DATA = [("EXIF:Make", "FUJIFILM", ["Tulips.jpg", "Tulips_edited.jpeg"])]
 
+UUID_LIVE_EDITED = "136A78FA-1B90-46CC-88A7-CCA3331F0353"  # IMG_4813.HEIC
+CLI_EXPORT_LIVE_EDITED = [
+    "IMG_4813.HEIC",
+    "IMG_4813.mov",
+    "IMG_4813_edited.jpeg",
+    "IMG_4813_edited.mov",
+]
+
 
 def modify_file(filename):
     """appends data to a file to modify it"""
@@ -1268,7 +1279,8 @@ def test_query_duplicate():
     runner = CliRunner()
     cwd = os.getcwd()
     result = runner.invoke(
-        query, ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--duplicate"],
+        query,
+        ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--duplicate"],
     )
     assert result.exit_code == 0
 
@@ -1289,7 +1301,8 @@ def test_query_location():
     runner = CliRunner()
     cwd = os.getcwd()
     result = runner.invoke(
-        query, ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--location"],
+        query,
+        ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--location"],
     )
     assert result.exit_code == 0
 
@@ -1311,7 +1324,8 @@ def test_query_no_location():
     runner = CliRunner()
     cwd = os.getcwd()
     result = runner.invoke(
-        query, ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--no-location"],
+        query,
+        ["--json", "--db", os.path.join(cwd, CLI_PHOTOS_DB), "--no-location"],
     )
     assert result.exit_code == 0
 
@@ -1432,6 +1446,7 @@ def test_export_uuid_from_file():
         files = glob.glob("*")
         assert sorted(files) == sorted(CLI_EXPORT_UUID_FROM_FILE_FILENAMES)
 
+
 def test_export_skip_uuid_from_file():
     """Test export with --skip-uuid-from-file"""
     import glob
@@ -1460,6 +1475,7 @@ def test_export_skip_uuid_from_file():
         for skipped_file in CLI_EXPORT_SKIP_UUID_FILENAMES:
             assert skipped_file not in files
 
+
 def test_export_skip_uuid():
     """Test export with --skip-uuid"""
     import glob
@@ -4304,7 +4320,7 @@ def test_export_error(monkeypatch):
 @pytest.mark.skipif(exiftool is None, reason="exiftool not installed")
 @pytest.mark.parametrize("exiftag,exifvalue,files_expected", EXPORT_EXIF_DATA)
 def test_export_exif(exiftag, exifvalue, files_expected):
-    """Test export --exif query """
+    """Test export --exif query"""
     import glob
     import os
     import os.path
@@ -4665,6 +4681,32 @@ def test_export_update_basic():
         )
 
 
+@pytest.mark.skipif(
+    "OSXPHOTOS_TEST_EXPORT" not in os.environ,
+    reason="Skip if not running on author's personal library.",
+)
+def test_export_live_edited():
+    """test export of edited live image #576"""
+    import glob
+    import os
+    import os.path
+
+    from osxphotos.cli import OSXPHOTOS_EXPORT_DB, export
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        # basic export
+        result = runner.invoke(
+            export,
+            [os.path.join(cwd, PHOTOS_DB_RHET), ".", "-V", "--uuid", UUID_LIVE_EDITED],
+        )
+        assert result.exit_code == 0
+        files = glob.glob("*")
+        assert sorted(files) == sorted(CLI_EXPORT_LIVE_EDITED)
+
+
 def test_export_update_child_folder():
     """test export then update into a child folder of previous export"""
     import glob
@@ -7642,6 +7684,7 @@ def test_export_query_function():
 def test_export_album_seq():
     """Test {album_seq} template"""
     import glob
+
     from osxphotos.cli import cli
 
     runner = CliRunner()
@@ -7719,7 +7762,6 @@ def test_export_description_template_conditional():
     import osxphotos
     from osxphotos.cli import cli
     from osxphotos.exiftool import ExifTool
-    import json
 
     runner = CliRunner()
     cwd = os.getcwd()