forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented #731, export_id in report database

Browse Source
This commit is contained in:
Rhet Turnbull
2022-07-23 10:11:39 -07:00
parent 337d422346
commit bd33b61882
34 changed files with 441 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.50.6"
__version__ = "0.50.7"

18
osxphotos/cli/report_writer.py
View File

@@ -2,16 +2,18 @@
import csv
import datetime
import json
import os
import os.path
import sqlite3
from abc import ABC, abstractmethod
from contextlib import suppress
from typing import Union, Dict
from typing import Dict, Union
from osxphotos.photoexporter import ExportResults
from osxphotos.export_db import OSXPHOTOS_ABOUT_STRING
from osxphotos.photoexporter import ExportResults
from osxphotos.sqlite_utils import sqlite_columns
__all__ = [
"report_writer_factory",
@@ -164,6 +166,8 @@ class ReportWriterSQLite(ReportWriterABC):
with suppress(FileNotFoundError):
os.unlink(self.output_file)
self.export_id = datetime.datetime.now().isoformat()
self._conn = sqlite3.connect(self.output_file)
self._create_tables()
@@ -172,12 +176,13 @@ class ReportWriterSQLite(ReportWriterABC):
all_results = prepare_results_for_writing(export_results)
for data in list(all_results.values()):
data["export_id"] = self.export_id
cursor = self._conn.cursor()
cursor.execute(
"INSERT INTO report "
"(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album) "
"(datetime, filename, exported, new, updated, skipped, exif_updated, touched, converted_to_jpeg, sidecar_xmp, sidecar_json, sidecar_exiftool, missing, error, exiftool_warning, exiftool_error, extended_attributes_written, extended_attributes_skipped, cleanup_deleted_file, cleanup_deleted_directory, exported_album, export_id) "
"VALUES "
"(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album);",
"(:datetime, :filename, :exported, :new, :updated, :skipped, :exif_updated, :touched, :converted_to_jpeg, :sidecar_xmp, :sidecar_json, :sidecar_exiftool, :missing, :error, :exiftool_warning, :exiftool_error, :extended_attributes_written, :extended_attributes_skipped, :cleanup_deleted_file, :cleanup_deleted_directory, :exported_album, :export_id);",
data,
)
self._conn.commit()
@@ -229,6 +234,11 @@ class ReportWriterSQLite(ReportWriterABC):
self._conn.commit()
# migrate report table to add export_id if needed (#731)
if "export_id" not in sqlite_columns(self._conn, "report"):
self._conn.cursor().execute("ALTER TABLE report ADD COLUMN export_id text;")
self._conn.commit()
def __del__(self):
with suppress(Exception):
self.close()

BIN
osxphotos/docs/docs.zip
View File

Binary file not shown.

5
osxphotos/photosdb/_photosdb_process_comments.py
View File

@@ -6,7 +6,8 @@ import datetime
from dataclasses import dataclass
from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION, TIME_DELTA
from ..utils import _open_sql_file, normalize_unicode
from ..utils import normalize_unicode
from ..sqlite_utils import sqlite_open_ro
def _process_comments(self):
@@ -65,7 +66,7 @@ def _process_comments_5(photosdb):
asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
(conn, cursor) = _open_sql_file(db)
(conn, cursor) = sqlite_open_ro(db)
results = conn.execute(
"""

4
osxphotos/photosdb/_photosdb_process_exif.py
View File

@@ -4,7 +4,7 @@
import logging
from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
from ..utils import _db_is_locked, _open_sql_file
from ..sqlite_utils import sqlite_open_ro, sqlite_db_is_locked
from .photosdb_utils import get_db_version
@@ -38,7 +38,7 @@ def _process_exifinfo_5(photosdb):
asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
(conn, cursor) = _open_sql_file(db)
(conn, cursor) = sqlite_open_ro(db)
result = conn.execute(
f"""

7
osxphotos/photosdb/_photosdb_process_faceinfo.py
View File

@@ -3,7 +3,8 @@
from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
from ..utils import _open_sql_file, normalize_unicode
from ..sqlite_utils import sqlite_open_ro
from ..utils import normalize_unicode
from .photosdb_utils import get_db_version
"""
@@ -40,7 +41,7 @@ def _process_faceinfo_4(photosdb):
"""
db = photosdb._tmp_db
(conn, cursor) = _open_sql_file(db)
(conn, cursor) = sqlite_open_ro(db)
result = cursor.execute(
"""
@@ -179,7 +180,7 @@ def _process_faceinfo_5(photosdb):
asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
(conn, cursor) = _open_sql_file(db)
(conn, cursor) = sqlite_open_ro(db)
result = cursor.execute(
f"""

4
osxphotos/photosdb/_photosdb_process_scoreinfo.py
View File

@@ -5,7 +5,7 @@
import logging
from .._constants import _DB_TABLE_NAMES, _PHOTOS_4_VERSION
from ..utils import _open_sql_file
from ..sqlite_utils import sqlite_open_ro
from .photosdb_utils import get_db_version
"""
@@ -48,7 +48,7 @@ def _process_scoreinfo_5(photosdb):
asset_table = _DB_TABLE_NAMES[photosdb._photos_ver]["ASSET"]
(conn, cursor) = _open_sql_file(db)
(conn, cursor) = sqlite_open_ro(db)
result = cursor.execute(
f"""

9
osxphotos/photosdb/_photosdb_process_searchinfo.py
View File

@@ -3,14 +3,15 @@
ref: https://github.com/dogsheep/photos-to-sqlite/issues/16
"""
from functools import lru_cache
import logging
import pathlib
import uuid as uuidlib
from functools import lru_cache
from pprint import pformat
from .._constants import _PHOTOS_4_VERSION, SEARCH_CATEGORY_LABEL
from ..utils import _db_is_locked, _open_sql_file, normalize_unicode
from ..sqlite_utils import sqlite_open_ro, sqlite_db_is_locked
from ..utils import normalize_unicode
"""
This module should be imported in the class defintion of PhotosDB in photosdb.py
@@ -63,12 +64,12 @@ def _process_searchinfo(self):
logging.warning(f"could not find search db: {search_db_path}")
return None
if _db_is_locked(search_db_path):
if sqlite_db_is_locked(search_db_path):
search_db = self._copy_db_file(search_db_path)
else:
search_db = search_db_path
(conn, c) = _open_sql_file(search_db)
(conn, c) = sqlite_open_ro(search_db)
result = c.execute(
"""

13
osxphotos/photosdb/photosdb.py
View File

@@ -57,11 +57,10 @@ from ..photoinfo import PhotoInfo
from ..phototemplate import RenderOptions
from ..queryoptions import QueryOptions
from ..rich_utils import add_rich_markup_tag
from ..sqlite_utils import sqlite_open_ro, sqlite_db_is_locked
from ..utils import (
_check_file_exists,
_db_is_locked,
_get_os_version,
_open_sql_file,
get_last_library_path,
noop,
normalize_unicode,
@@ -309,7 +308,7 @@ class PhotosDB:
# Photos maintains an exclusive lock on the database file while Photos is open
# photoanalysisd sometimes maintains this lock even after Photos is closed
# In those cases, make a temp copy of the file for sqlite3 to read
if _db_is_locked(self._dbfile):
if sqlite_db_is_locked(self._dbfile):
verbose(f"Database locked, creating temporary copy.")
self._tmp_db = self._copy_db_file(self._dbfile)
@@ -325,7 +324,7 @@ class PhotosDB:
self._dbfile_actual = self._tmp_db = dbfile
verbose(f"Processing database {self._filepath(self._dbfile_actual)}")
# if database is exclusively locked, make a copy of it and use the copy
if _db_is_locked(self._dbfile_actual):
if sqlite_db_is_locked(self._dbfile_actual):
verbose(f"Database locked, creating temporary copy.")
self._tmp_db = self._copy_db_file(self._dbfile_actual)
@@ -578,7 +577,7 @@ class PhotosDB:
Returns:
tuple of (connection, cursor) to sqlite3 database
"""
return _open_sql_file(self._tmp_db)
return sqlite_open_ro(self._tmp_db)
def _copy_db_file(self, fname):
"""copies the sqlite database file to a temp file"""
@@ -642,7 +641,7 @@ class PhotosDB:
self._photos_ver = 4 # only used in Photos 5+
(conn, c) = _open_sql_file(self._tmp_db)
(conn, c) = sqlite_open_ro(self._tmp_db)
# get info to associate persons with photos
# then get detected faces in each photo and link to persons
@@ -1593,7 +1592,7 @@ class PhotosDB:
logging.debug(f"_process_database5")
verbose = self._verbose
verbose(f"Processing database.")
(conn, c) = _open_sql_file(self._tmp_db)
(conn, c) = sqlite_open_ro(self._tmp_db)
# some of the tables/columns have different names in different versions of Photos
photos_ver = get_db_model_version(self._tmp_db)

6
osxphotos/photosdb/photosdb_utils.py
View File

@@ -15,7 +15,7 @@ from .._constants import (
_PHOTOS_8_MODEL_VERSION,
_TESTED_DB_VERSIONS,
)
from ..utils import _open_sql_file
from ..sqlite_utils import sqlite_open_ro
__all__ = [
"get_db_version",
@@ -37,7 +37,7 @@ def get_db_version(db_file):
version = None
(conn, c) = _open_sql_file(db_file)
(conn, c) = sqlite_open_ro(db_file)
# get database version
c.execute("SELECT value from LiGlobals where LiGlobals.keyPath is 'libraryVersion'")
@@ -64,7 +64,7 @@ def get_model_version(db_file):
version = None
(conn, c) = _open_sql_file(db_file)
(conn, c) = sqlite_open_ro(db_file)
# get database version
c.execute("SELECT MAX(Z_VERSION), Z_PLIST FROM Z_METADATA")

57
osxphotos/sqlite_utils.py Normal file
View File

@@ -0,0 +1,57 @@
"""sqlite utils for use by osxphotos"""
import os.path
import pathlib
import sqlite3
from typing import List, Tuple
def sqlite_open_ro(dbname: str) -> Tuple[sqlite3.Connection, sqlite3.Cursor]:
"""opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor)"""
try:
dbpath = pathlib.Path(dbname).resolve()
conn = sqlite3.connect(f"{dbpath.as_uri()}?mode=ro", timeout=1, uri=True)
c = conn.cursor()
except sqlite3.Error as e:
raise sqlite3.Error(
f"An error occurred opening sqlite file: {e} {dbname}"
) from e
return (conn, c)
def sqlite_db_is_locked(dbname):
"""check to see if a sqlite3 db is locked
returns True if database is locked, otherwise False
dbname: name of database to test"""
# first, check to see if lock file exists, if so, assume the file is locked
lock_name = f"{dbname}.lock"
if os.path.exists(lock_name):
return True
# no lock file so try to read from the database to see if it's locked
locked = None
try:
(conn, c) = sqlite_open_ro(dbname)
c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
conn.close()
locked = False
except Exception:
locked = True
return locked
def sqlite_tables(conn: sqlite3.Connection) -> List[str]:
"""Returns list of tables found in sqlite db"""
results = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table';"
).fetchall()
return [row[0] for row in results]
def sqlite_columns(conn: sqlite3.Connection, table: str) -> List[str]:
"""Returns list of column names found in table in sqlite database"""
results = conn.execute(f"PRAGMA table_info({table});")
return [row[1] for row in results]

40
osxphotos/utils.py
View File

@@ -2,7 +2,6 @@
import datetime
import fnmatch
import glob
import hashlib
import importlib
import inspect
@@ -12,7 +11,6 @@ import os.path
import pathlib
import platform
import re
import sqlite3
import subprocess
import sys
import unicodedata
@@ -363,44 +361,6 @@ def list_directory(
return files
def _open_sql_file(dbname):
"""opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor)"""
try:
dbpath = pathlib.Path(dbname).resolve()
conn = sqlite3.connect(f"{dbpath.as_uri()}?mode=ro", timeout=1, uri=True)
c = conn.cursor()
except sqlite3.Error as e:
sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {dbname}")
return (conn, c)
def _db_is_locked(dbname):
"""check to see if a sqlite3 db is locked
returns True if database is locked, otherwise False
dbname: name of database to test"""
# first, check to see if lock file exists, if so, assume the file is locked
lock_name = f"{dbname}.lock"
if os.path.exists(lock_name):
logging.debug(f"{dbname} is locked")
return True
# no lock file so try to read from the database to see if it's locked
locked = None
try:
(conn, c) = _open_sql_file(dbname)
c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
conn.close()
logging.debug(f"{dbname} is not locked")
locked = False
except:
logging.debug(f"{dbname} is locked")
locked = True
return locked
def normalize_unicode(value):
"""normalize unicode data"""
if value is None: