Updated PhotosDB to only copy database if locked, speed improvement for cases where DB not locked; closes #34

This commit is contained in:
Rhet Turnbull 2020-01-30 05:38:11 -08:00
parent 27994c9fd3
commit ac8be51156
155 changed files with 717 additions and 27 deletions

View File

@ -25,12 +25,19 @@ from ._constants import (
) )
from ._version import __version__ from ._version import __version__
from .photoinfo import PhotoInfo from .photoinfo import PhotoInfo
from .utils import _check_file_exists, _get_os_version, get_last_library_path, _debug from .utils import (
_check_file_exists,
_get_os_version,
get_last_library_path,
_debug,
_open_sql_file,
_db_is_locked,
)
# TODO: Add test for imageTimeZoneOffsetSeconds = None # TODO: Add test for imageTimeZoneOffsetSeconds = None
# TODO: Fix command line so multiple --keyword, etc. are AND (instead of OR as they are in .photos()) # TODO: Fix command line so multiple --keyword, etc. are AND (instead of OR as they are in .photos())
# Or fix the help text to match behavior # Or fix the help text to match behavior
# TODO: Add test for __str__ and to_json # TODO: Add test for __str__
# TODO: Add special albums and magic albums # TODO: Add special albums and magic albums
@ -62,7 +69,7 @@ class PhotosDB:
# Path to the Photos library database file # Path to the Photos library database file
self._dbfile = None self._dbfile = None
# the actual file with library data which on Photos 5 is Photos.sqlite instead of photos.db # the actual file with library data, which on Photos 5 is Photos.sqlite instead of photos.db
self._dbfile_actual = None self._dbfile_actual = None
# Dict with information about all photos by uuid # Dict with information about all photos by uuid
self._dbphotos = {} self._dbphotos = {}
@ -94,7 +101,8 @@ class PhotosDB:
if dbfile: if dbfile:
# shouldn't pass via both *args and dbfile= # shouldn't pass via both *args and dbfile=
raise TypeError( raise TypeError(
f"photos database path must be specified as argument or named parameter dbfile but not both: args: {dbfile_}, dbfile: {dbfile}", f"photos database path must be specified as argument or "
f"named parameter dbfile but not both: args: {dbfile_}, dbfile: {dbfile}",
dbfile_, dbfile_,
dbfile, dbfile,
) )
@ -123,22 +131,38 @@ class PhotosDB:
if _debug(): if _debug():
logging.debug(f"dbfile = {dbfile}") logging.debug(f"dbfile = {dbfile}")
self._dbfile = self._dbfile_actual = os.path.abspath(dbfile) # init database names
# _tmp_db is the file that will processed by _process_database4/5
# assume _tmp_db will be _dbfile or _dbfile_actual based on Photos version
# unless DB is locked, in which case _tmp_db will point to a temporary copy
# if Photos <=4, _dbfile = _dbfile_actual = photos.db
# if Photos >= 5, _dbfile = photos.db, from which we get DB version but the actual
# photos data is in Photos.sqlite
# In either case, a temporary copy will be made if the DB is locked by Photos
# or photosanalysisd
self._dbfile = self._dbfile_actual = self._tmp_db = os.path.abspath(dbfile)
# if database is exclusively locked, make a copy of it and use the copy
# Photos maintains an exclusive lock on the database file while Photos is open
# photoanalysisd sometimes maintains this lock even after Photos is closed
# In those cases, make a temp copy of the file for sqlite3 to read
if _db_is_locked(self._dbfile):
self._tmp_db = self._copy_db_file(self._dbfile)
self._tmp_db = self._copy_db_file(self._dbfile)
self._db_version = self._get_db_version() self._db_version = self._get_db_version()
# If Photos >= 5, actual data isn't in photos.db but in Photos.sqlite # If Photos >= 5, actual data isn't in photos.db but in Photos.sqlite
if int(self._db_version) >= int(_PHOTOS_5_VERSION): if int(self._db_version) >= int(_PHOTOS_5_VERSION):
if _debug():
logging.debug(f"version is {self._db_version}")
dbpath = pathlib.Path(self._dbfile).parent dbpath = pathlib.Path(self._dbfile).parent
dbfile = dbpath / "Photos.sqlite" dbfile = dbpath / "Photos.sqlite"
if not _check_file_exists(dbfile): if not _check_file_exists(dbfile):
sys.exit(f"dbfile {dbfile} does not exist") raise FileNotFoundError(f"dbfile {dbfile} does not exist", dbfile)
else: else:
self._tmp_db = self._copy_db_file(dbfile) self._dbfile_actual = self._tmp_db = dbfile
self._dbfile_actual = dbfile # if database is exclusively locked, make a copy of it and use the copy
if _db_is_locked(self._dbfile_actual):
self._tmp_db = self._copy_db_file(self._dbfile_actual)
if _debug(): if _debug():
logging.debug( logging.debug(
f"_dbfile = {self._dbfile}, _dbfile_actual = {self._dbfile_actual}" f"_dbfile = {self._dbfile}, _dbfile_actual = {self._dbfile_actual}"
@ -319,21 +343,24 @@ class PhotosDB:
return dest_path return dest_path
def _open_sql_file(self, fname): # def _open_sql_file(self, fname):
""" opens sqlite file fname and returns connection to the database """ # """ opens sqlite file fname in read-only mode
try: # returns tuple of (connection, cursor) """
conn = sqlite3.connect(f"{pathlib.Path(fname).as_uri()}?mode=ro", uri=True) # try:
c = conn.cursor() # conn = sqlite3.connect(
except sqlite3.Error as e: # f"{pathlib.Path(fname).as_uri()}?mode=ro", timeout=1, uri=True
sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {fname}") # )
return (conn, c) # c = conn.cursor()
# except sqlite3.Error as e:
# sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {fname}")
# return (conn, c)
def _get_db_version(self): def _get_db_version(self):
""" gets the Photos DB version from LiGlobals table """ """ gets the Photos DB version from LiGlobals table """
""" returns the version as str""" """ returns the version as str"""
version = None version = None
(conn, c) = self._open_sql_file(self._tmp_db) (conn, c) = _open_sql_file(self._tmp_db)
# get database version # get database version
c.execute( c.execute(
@ -358,7 +385,7 @@ class PhotosDB:
# Epoch is Jan 1, 2001 # Epoch is Jan 1, 2001
td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds() td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
(conn, c) = self._open_sql_file(self._tmp_db) (conn, c) = _open_sql_file(self._tmp_db)
# Look for all combinations of persons and pictures # Look for all combinations of persons and pictures
c.execute( c.execute(
@ -799,7 +826,7 @@ class PhotosDB:
# Epoch is Jan 1, 2001 # Epoch is Jan 1, 2001
td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds() td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
(conn, c) = self._open_sql_file(self._tmp_db) (conn, c) = _open_sql_file(self._tmp_db)
# Look for all combinations of persons and pictures # Look for all combinations of persons and pictures
if _debug(): if _debug():

View File

@ -2,10 +2,11 @@ import glob
import logging import logging
import os.path import os.path
import platform import platform
import sqlite3
import subprocess import subprocess
import tempfile import tempfile
import urllib.parse import urllib.parse
from pathlib import Path import pathlib
from plistlib import load as plistload from plistlib import load as plistload
import CoreFoundation import CoreFoundation
@ -177,8 +178,8 @@ def get_system_library_path():
) )
return None return None
plist_file = Path( plist_file = pathlib.Path(
str(Path.home()) str(pathlib.Path.home())
+ "/Library/Containers/com.apple.photolibraryd/Data/Library/Preferences/com.apple.photolibraryd.plist" + "/Library/Containers/com.apple.photolibraryd/Data/Library/Preferences/com.apple.photolibraryd.plist"
) )
if plist_file.is_file(): if plist_file.is_file():
@ -200,8 +201,8 @@ def get_system_library_path():
def get_last_library_path(): def get_last_library_path():
""" returns the path to the last opened Photos library """ returns the path to the last opened Photos library
If a library has never been opened, returns None """ If a library has never been opened, returns None """
plist_file = Path( plist_file = pathlib.Path(
str(Path.home()) str(pathlib.Path.home())
+ "/Library/Containers/com.apple.Photos/Data/Library/Preferences/com.apple.Photos.plist" + "/Library/Containers/com.apple.Photos/Data/Library/Preferences/com.apple.Photos.plist"
) )
if plist_file.is_file(): if plist_file.is_file():
@ -376,3 +377,41 @@ def _export_photo_uuid_applescript(
return new_path return new_path
else: else:
return None return None
def _open_sql_file(dbname):
""" opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor) """
try:
dbpath = pathlib.Path(dbname).resolve()
conn = sqlite3.connect(f"{dbpath.as_uri()}?mode=ro", timeout=1, uri=True)
c = conn.cursor()
except sqlite3.Error as e:
sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {dbname}")
return (conn, c)
def _db_is_locked(dbname):
""" check to see if a sqlite3 db is locked
returns True if database is locked, otherwise False
dbname: name of database to test """
# first, check to see if lock file exists, if so, assume the file is locked
lock_name = f"{dbname}.lock"
if os.path.exists(lock_name):
logging.debug(f"{dbname} is locked")
return True
# no lock file so try to read from the database to see if it's locked
locked = None
try:
(conn, c) = _open_sql_file(dbname)
c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
conn.close()
logging.debug(f"{dbname} is not locked")
locked = False
except Exception as e:
logging.debug(f"{dbname} is locked")
locked = True
return locked

Binary file not shown.

After

Width:  |  Height:  |  Size: 453 KiB

View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>DatabaseMinorVersion</key>
<integer>1</integer>
<key>DatabaseVersion</key>
<integer>112</integer>
<key>LastOpenMode</key>
<integer>2</integer>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
<key>MetaSchemaVersion</key>
<integer>2</integer>
<key>createDate</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</plist>

Binary file not shown.

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>hostname</key>
<string>Rhet-and-Sarahs-iMac.local</string>
<key>hostuuid</key>
<string>A91B9F4C-F77A-565D-A8E1-B550C8F012AF</string>
<key>pid</key>
<integer>741</integer>
<key>processname</key>
<string>photolibraryd</string>
<key>uid</key>
<integer>501</integer>
</dict>
</plist>

View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LithiumMessageTracer</key>
<dict>
<key>LastReportedDate</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</dict>
</plist>

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
<date>2020-01-30T00:26:32Z</date>
<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
<date>2020-01-30T00:26:32Z</date>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PLLanguageAndLocaleKey</key>
<string>en-US:en_US</string>
<key>PLLastGeoProviderIdKey</key>
<string>7618</string>
<key>PLLastLocationInfoFormatVer</key>
<integer>12</integer>
<key>PLLastRevGeoForcedProviderOutOfDateCheckVersionKey</key>
<integer>1</integer>
<key>PLLastRevGeoVerFileFetchDateKey</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</plist>

View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LastHistoryRowId</key>
<integer>83</integer>
<key>LibraryBuildTag</key>
<string>FF4B0E6D-6231-4212-91F6-7397F96959BD</string>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
</dict>
</plist>

View File

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>FileVersion</key>
<integer>11</integer>
<key>Source</key>
<dict>
<key>35230</key>
<dict>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>1</integer>
</dict>
<key>CurrentVersion</key>
<integer>1</integer>
<key>NoResultErrorIsSuccess</key>
<true/>
</dict>
<key>57879</key>
<dict>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>1</integer>
</dict>
<key>CurrentVersion</key>
<integer>1</integer>
<key>NoResultErrorIsSuccess</key>
<true/>
</dict>
<key>7618</key>
<dict>
<key>AddCountyIfNeeded</key>
<true/>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>10</integer>
</dict>
<key>CurrentVersion</key>
<integer>10</integer>
</dict>
</dict>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 178 KiB

View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>DatabaseMinorVersion</key>
<integer>1</integer>
<key>DatabaseVersion</key>
<integer>112</integer>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
<key>MetaSchemaVersion</key>
<integer>2</integer>
<key>SnapshotComplete</key>
<true/>
<key>SnapshotCompletedDate</key>
<date>2020-01-30T00:26:31Z</date>
<key>SnapshotTables</key>
<dict/>
</dict>
</plist>

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LibrarySchemaVersion</key>
<integer>5001</integer>
<key>MetaSchemaVersion</key>
<integer>3</integer>
</dict>
</plist>

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>hostname</key>
<string>Rhets-MacBook-Pro.local</string>
<key>hostuuid</key>
<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
<key>pid</key>
<integer>1309</integer>
<key>processname</key>
<string>photolibraryd</string>
<key>uid</key>
<integer>501</integer>
</dict>
</plist>

View File

@ -0,0 +1,188 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>BlacklistedMeaningsByMeaning</key>
<dict/>
<key>MePersonUUID</key>
<string>39488755-78C0-40B2-B378-EDA280E1823C</string>
<key>SceneWhitelist</key>
<array>
<string>Graduation</string>
<string>Aquarium</string>
<string>Food</string>
<string>Ice Skating</string>
<string>Mountain</string>
<string>Cliff</string>
<string>Basketball</string>
<string>Tennis</string>
<string>Jewelry</string>
<string>Cheese</string>
<string>Softball</string>
<string>Football</string>
<string>Circus</string>
<string>Jet Ski</string>
<string>Playground</string>
<string>Carousel</string>
<string>Paint Ball</string>
<string>Windsurfing</string>
<string>Sailboat</string>
<string>Sunbathing</string>
<string>Dam</string>
<string>Fireplace</string>
<string>Flower</string>
<string>Scuba</string>
<string>Hiking</string>
<string>Cetacean</string>
<string>Pier</string>
<string>Bowling</string>
<string>Snowboarding</string>
<string>Zoo</string>
<string>Snowmobile</string>
<string>Theater</string>
<string>Boat</string>
<string>Casino</string>
<string>Car</string>
<string>Diving</string>
<string>Cycling</string>
<string>Musical Instrument</string>
<string>Board Game</string>
<string>Castle</string>
<string>Sunset Sunrise</string>
<string>Martial Arts</string>
<string>Motocross</string>
<string>Submarine</string>
<string>Cat</string>
<string>Snow</string>
<string>Kiteboarding</string>
<string>Squash</string>
<string>Geyser</string>
<string>Music</string>
<string>Archery</string>
<string>Desert</string>
<string>Blackjack</string>
<string>Fireworks</string>
<string>Sportscar</string>
<string>Feline</string>
<string>Soccer</string>
<string>Museum</string>
<string>Baby</string>
<string>Fencing</string>
<string>Railroad</string>
<string>Nascar</string>
<string>Sky Surfing</string>
<string>Bird</string>
<string>Games</string>
<string>Baseball</string>
<string>Dressage</string>
<string>Snorkeling</string>
<string>Pyramid</string>
<string>Kite</string>
<string>Rowboat</string>
<string>Golf</string>
<string>Watersports</string>
<string>Lightning</string>
<string>Canyon</string>
<string>Auditorium</string>
<string>Night Sky</string>
<string>Karaoke</string>
<string>Skiing</string>
<string>Parade</string>
<string>Forest</string>
<string>Hot Air Balloon</string>
<string>Dragon Parade</string>
<string>Easter Egg</string>
<string>Monument</string>
<string>Jungle</string>
<string>Thanksgiving</string>
<string>Jockey Horse</string>
<string>Stadium</string>
<string>Airplane</string>
<string>Ballet</string>
<string>Yoga</string>
<string>Coral Reef</string>
<string>Skating</string>
<string>Wrestling</string>
<string>Bicycle</string>
<string>Tattoo</string>
<string>Amusement Park</string>
<string>Canoe</string>
<string>Cheerleading</string>
<string>Ping Pong</string>
<string>Fishing</string>
<string>Magic</string>
<string>Reptile</string>
<string>Winter Sport</string>
<string>Waterfall</string>
<string>Train</string>
<string>Bonsai</string>
<string>Surfing</string>
<string>Dog</string>
<string>Cake</string>
<string>Sledding</string>
<string>Sandcastle</string>
<string>Glacier</string>
<string>Lighthouse</string>
<string>Equestrian</string>
<string>Rafting</string>
<string>Shore</string>
<string>Hockey</string>
<string>Santa Claus</string>
<string>Formula One Car</string>
<string>Sport</string>
<string>Vehicle</string>
<string>Boxing</string>
<string>Rollerskating</string>
<string>Underwater</string>
<string>Orchestra</string>
<string>Carnival</string>
<string>Rocket</string>
<string>Skateboarding</string>
<string>Helicopter</string>
<string>Performance</string>
<string>Oktoberfest</string>
<string>Water Polo</string>
<string>Skate Park</string>
<string>Animal</string>
<string>Nightclub</string>
<string>String Instrument</string>
<string>Dinosaur</string>
<string>Gymnastics</string>
<string>Cricket</string>
<string>Volcano</string>
<string>Lake</string>
<string>Aurora</string>
<string>Dancing</string>
<string>Concert</string>
<string>Rock Climbing</string>
<string>Hang Glider</string>
<string>Rodeo</string>
<string>Fish</string>
<string>Art</string>
<string>Motorcycle</string>
<string>Volleyball</string>
<string>Wake Boarding</string>
<string>Badminton</string>
<string>Motor Sport</string>
<string>Sumo</string>
<string>Parasailing</string>
<string>Skydiving</string>
<string>Kickboxing</string>
<string>Pinata</string>
<string>Foosball</string>
<string>Go Kart</string>
<string>Poker</string>
<string>Kayak</string>
<string>Swimming</string>
<string>Atv</string>
<string>Beach</string>
<string>Dartboard</string>
<string>Athletics</string>
<string>Camping</string>
<string>Tornado</string>
<string>Billiards</string>
<string>Rugby</string>
<string>Airshow</string>
</array>
</dict>
</plist>

View File

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>insertAlbum</key>
<array/>
<key>insertAsset</key>
<array/>
<key>insertHighlight</key>
<array/>
<key>insertMemory</key>
<array/>
<key>insertMoment</key>
<array/>
<key>removeAlbum</key>
<array/>
<key>removeAsset</key>
<array/>
<key>removeHighlight</key>
<array/>
<key>removeMemory</key>
<array/>
<key>removeMoment</key>
<array/>
</dict>
</plist>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>embeddingVersion</key>
<string>1</string>
<key>localeIdentifier</key>
<string>en_US</string>
<key>sceneTaxonomySHA</key>
<string>87914a047c69fbe8013fad2c70fa70c6c03b08b56190fe4054c880e6b9f57cc3</string>
<key>searchIndexVersion</key>
<string>10</string>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 453 KiB

View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CollapsedSidebarSectionIdentifiers</key>
<array/>
</dict>
</plist>

Some files were not shown because too many files have changed in this diff Show More