Updated PhotosDB to only copy database if locked, speed improvement for cases where DB not locked; closes #34

This commit is contained in:
Rhet Turnbull 2020-01-30 05:38:11 -08:00
parent 27994c9fd3
commit ac8be51156
155 changed files with 717 additions and 27 deletions

View File

@ -25,12 +25,19 @@ from ._constants import (
)
from ._version import __version__
from .photoinfo import PhotoInfo
from .utils import _check_file_exists, _get_os_version, get_last_library_path, _debug
from .utils import (
_check_file_exists,
_get_os_version,
get_last_library_path,
_debug,
_open_sql_file,
_db_is_locked,
)
# TODO: Add test for imageTimeZoneOffsetSeconds = None
# TODO: Fix command line so multiple --keyword, etc. are AND (instead of OR as they are in .photos())
# Or fix the help text to match behavior
# TODO: Add test for __str__ and to_json
# TODO: Add test for __str__
# TODO: Add special albums and magic albums
@ -62,7 +69,7 @@ class PhotosDB:
# Path to the Photos library database file
self._dbfile = None
# the actual file with library data which on Photos 5 is Photos.sqlite instead of photos.db
# the actual file with library data, which on Photos 5 is Photos.sqlite instead of photos.db
self._dbfile_actual = None
# Dict with information about all photos by uuid
self._dbphotos = {}
@ -94,7 +101,8 @@ class PhotosDB:
if dbfile:
# shouldn't pass via both *args and dbfile=
raise TypeError(
f"photos database path must be specified as argument or named parameter dbfile but not both: args: {dbfile_}, dbfile: {dbfile}",
f"photos database path must be specified as argument or "
f"named parameter dbfile but not both: args: {dbfile_}, dbfile: {dbfile}",
dbfile_,
dbfile,
)
@ -123,22 +131,38 @@ class PhotosDB:
if _debug():
logging.debug(f"dbfile = {dbfile}")
self._dbfile = self._dbfile_actual = os.path.abspath(dbfile)
# init database names
# _tmp_db is the file that will processed by _process_database4/5
# assume _tmp_db will be _dbfile or _dbfile_actual based on Photos version
# unless DB is locked, in which case _tmp_db will point to a temporary copy
# if Photos <=4, _dbfile = _dbfile_actual = photos.db
# if Photos >= 5, _dbfile = photos.db, from which we get DB version but the actual
# photos data is in Photos.sqlite
# In either case, a temporary copy will be made if the DB is locked by Photos
# or photosanalysisd
self._dbfile = self._dbfile_actual = self._tmp_db = os.path.abspath(dbfile)
# if database is exclusively locked, make a copy of it and use the copy
# Photos maintains an exclusive lock on the database file while Photos is open
# photoanalysisd sometimes maintains this lock even after Photos is closed
# In those cases, make a temp copy of the file for sqlite3 to read
if _db_is_locked(self._dbfile):
self._tmp_db = self._copy_db_file(self._dbfile)
self._tmp_db = self._copy_db_file(self._dbfile)
self._db_version = self._get_db_version()
# If Photos >= 5, actual data isn't in photos.db but in Photos.sqlite
if int(self._db_version) >= int(_PHOTOS_5_VERSION):
if _debug():
logging.debug(f"version is {self._db_version}")
dbpath = pathlib.Path(self._dbfile).parent
dbfile = dbpath / "Photos.sqlite"
if not _check_file_exists(dbfile):
sys.exit(f"dbfile {dbfile} does not exist")
raise FileNotFoundError(f"dbfile {dbfile} does not exist", dbfile)
else:
self._tmp_db = self._copy_db_file(dbfile)
self._dbfile_actual = dbfile
self._dbfile_actual = self._tmp_db = dbfile
# if database is exclusively locked, make a copy of it and use the copy
if _db_is_locked(self._dbfile_actual):
self._tmp_db = self._copy_db_file(self._dbfile_actual)
if _debug():
logging.debug(
f"_dbfile = {self._dbfile}, _dbfile_actual = {self._dbfile_actual}"
@ -319,21 +343,24 @@ class PhotosDB:
return dest_path
def _open_sql_file(self, fname):
""" opens sqlite file fname and returns connection to the database """
try:
conn = sqlite3.connect(f"{pathlib.Path(fname).as_uri()}?mode=ro", uri=True)
c = conn.cursor()
except sqlite3.Error as e:
sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {fname}")
return (conn, c)
# def _open_sql_file(self, fname):
# """ opens sqlite file fname in read-only mode
# returns tuple of (connection, cursor) """
# try:
# conn = sqlite3.connect(
# f"{pathlib.Path(fname).as_uri()}?mode=ro", timeout=1, uri=True
# )
# c = conn.cursor()
# except sqlite3.Error as e:
# sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {fname}")
# return (conn, c)
def _get_db_version(self):
""" gets the Photos DB version from LiGlobals table """
""" returns the version as str"""
version = None
(conn, c) = self._open_sql_file(self._tmp_db)
(conn, c) = _open_sql_file(self._tmp_db)
# get database version
c.execute(
@ -358,7 +385,7 @@ class PhotosDB:
# Epoch is Jan 1, 2001
td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
(conn, c) = self._open_sql_file(self._tmp_db)
(conn, c) = _open_sql_file(self._tmp_db)
# Look for all combinations of persons and pictures
c.execute(
@ -799,7 +826,7 @@ class PhotosDB:
# Epoch is Jan 1, 2001
td = (datetime(2001, 1, 1, 0, 0) - datetime(1970, 1, 1, 0, 0)).total_seconds()
(conn, c) = self._open_sql_file(self._tmp_db)
(conn, c) = _open_sql_file(self._tmp_db)
# Look for all combinations of persons and pictures
if _debug():

View File

@ -2,10 +2,11 @@ import glob
import logging
import os.path
import platform
import sqlite3
import subprocess
import tempfile
import urllib.parse
from pathlib import Path
import pathlib
from plistlib import load as plistload
import CoreFoundation
@ -177,8 +178,8 @@ def get_system_library_path():
)
return None
plist_file = Path(
str(Path.home())
plist_file = pathlib.Path(
str(pathlib.Path.home())
+ "/Library/Containers/com.apple.photolibraryd/Data/Library/Preferences/com.apple.photolibraryd.plist"
)
if plist_file.is_file():
@ -200,8 +201,8 @@ def get_system_library_path():
def get_last_library_path():
""" returns the path to the last opened Photos library
If a library has never been opened, returns None """
plist_file = Path(
str(Path.home())
plist_file = pathlib.Path(
str(pathlib.Path.home())
+ "/Library/Containers/com.apple.Photos/Data/Library/Preferences/com.apple.Photos.plist"
)
if plist_file.is_file():
@ -376,3 +377,41 @@ def _export_photo_uuid_applescript(
return new_path
else:
return None
def _open_sql_file(dbname):
""" opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor) """
try:
dbpath = pathlib.Path(dbname).resolve()
conn = sqlite3.connect(f"{dbpath.as_uri()}?mode=ro", timeout=1, uri=True)
c = conn.cursor()
except sqlite3.Error as e:
sys.exit(f"An error occurred opening sqlite file: {e.args[0]} {dbname}")
return (conn, c)
def _db_is_locked(dbname):
""" check to see if a sqlite3 db is locked
returns True if database is locked, otherwise False
dbname: name of database to test """
# first, check to see if lock file exists, if so, assume the file is locked
lock_name = f"{dbname}.lock"
if os.path.exists(lock_name):
logging.debug(f"{dbname} is locked")
return True
# no lock file so try to read from the database to see if it's locked
locked = None
try:
(conn, c) = _open_sql_file(dbname)
c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
conn.close()
logging.debug(f"{dbname} is not locked")
locked = False
except Exception as e:
logging.debug(f"{dbname} is locked")
locked = True
return locked

Binary file not shown.

After

Width:  |  Height:  |  Size: 453 KiB

View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>DatabaseMinorVersion</key>
<integer>1</integer>
<key>DatabaseVersion</key>
<integer>112</integer>
<key>LastOpenMode</key>
<integer>2</integer>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
<key>MetaSchemaVersion</key>
<integer>2</integer>
<key>createDate</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</plist>

Binary file not shown.

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>hostname</key>
<string>Rhet-and-Sarahs-iMac.local</string>
<key>hostuuid</key>
<string>A91B9F4C-F77A-565D-A8E1-B550C8F012AF</string>
<key>pid</key>
<integer>741</integer>
<key>processname</key>
<string>photolibraryd</string>
<key>uid</key>
<integer>501</integer>
</dict>
</plist>

View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LithiumMessageTracer</key>
<dict>
<key>LastReportedDate</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</dict>
</plist>

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate</key>
<date>2020-01-30T00:26:32Z</date>
<key>PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate</key>
<date>2020-01-30T00:26:32Z</date>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PLLanguageAndLocaleKey</key>
<string>en-US:en_US</string>
<key>PLLastGeoProviderIdKey</key>
<string>7618</string>
<key>PLLastLocationInfoFormatVer</key>
<integer>12</integer>
<key>PLLastRevGeoForcedProviderOutOfDateCheckVersionKey</key>
<integer>1</integer>
<key>PLLastRevGeoVerFileFetchDateKey</key>
<date>2020-01-30T00:26:31Z</date>
</dict>
</plist>

View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LastHistoryRowId</key>
<integer>83</integer>
<key>LibraryBuildTag</key>
<string>FF4B0E6D-6231-4212-91F6-7397F96959BD</string>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
</dict>
</plist>

View File

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>FileVersion</key>
<integer>11</integer>
<key>Source</key>
<dict>
<key>35230</key>
<dict>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>1</integer>
</dict>
<key>CurrentVersion</key>
<integer>1</integer>
<key>NoResultErrorIsSuccess</key>
<true/>
</dict>
<key>57879</key>
<dict>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>1</integer>
</dict>
<key>CurrentVersion</key>
<integer>1</integer>
<key>NoResultErrorIsSuccess</key>
<true/>
</dict>
<key>7618</key>
<dict>
<key>AddCountyIfNeeded</key>
<true/>
<key>CountryMinVersions</key>
<dict>
<key>OTHER</key>
<integer>10</integer>
</dict>
<key>CurrentVersion</key>
<integer>10</integer>
</dict>
</dict>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 178 KiB

View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>DatabaseMinorVersion</key>
<integer>1</integer>
<key>DatabaseVersion</key>
<integer>112</integer>
<key>LibrarySchemaVersion</key>
<integer>2622</integer>
<key>MetaSchemaVersion</key>
<integer>2</integer>
<key>SnapshotComplete</key>
<true/>
<key>SnapshotCompletedDate</key>
<date>2020-01-30T00:26:31Z</date>
<key>SnapshotTables</key>
<dict/>
</dict>
</plist>

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>LibrarySchemaVersion</key>
<integer>5001</integer>
<key>MetaSchemaVersion</key>
<integer>3</integer>
</dict>
</plist>

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>hostname</key>
<string>Rhets-MacBook-Pro.local</string>
<key>hostuuid</key>
<string>9575E48B-8D5F-5654-ABAC-4431B1167324</string>
<key>pid</key>
<integer>1309</integer>
<key>processname</key>
<string>photolibraryd</string>
<key>uid</key>
<integer>501</integer>
</dict>
</plist>

View File

@ -0,0 +1,188 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>BlacklistedMeaningsByMeaning</key>
<dict/>
<key>MePersonUUID</key>
<string>39488755-78C0-40B2-B378-EDA280E1823C</string>
<key>SceneWhitelist</key>
<array>
<string>Graduation</string>
<string>Aquarium</string>
<string>Food</string>
<string>Ice Skating</string>
<string>Mountain</string>
<string>Cliff</string>
<string>Basketball</string>
<string>Tennis</string>
<string>Jewelry</string>
<string>Cheese</string>
<string>Softball</string>
<string>Football</string>
<string>Circus</string>
<string>Jet Ski</string>
<string>Playground</string>
<string>Carousel</string>
<string>Paint Ball</string>
<string>Windsurfing</string>
<string>Sailboat</string>
<string>Sunbathing</string>
<string>Dam</string>
<string>Fireplace</string>
<string>Flower</string>
<string>Scuba</string>
<string>Hiking</string>
<string>Cetacean</string>
<string>Pier</string>
<string>Bowling</string>
<string>Snowboarding</string>
<string>Zoo</string>
<string>Snowmobile</string>
<string>Theater</string>
<string>Boat</string>
<string>Casino</string>
<string>Car</string>
<string>Diving</string>
<string>Cycling</string>
<string>Musical Instrument</string>
<string>Board Game</string>
<string>Castle</string>
<string>Sunset Sunrise</string>
<string>Martial Arts</string>
<string>Motocross</string>
<string>Submarine</string>
<string>Cat</string>
<string>Snow</string>
<string>Kiteboarding</string>
<string>Squash</string>
<string>Geyser</string>
<string>Music</string>
<string>Archery</string>
<string>Desert</string>
<string>Blackjack</string>
<string>Fireworks</string>
<string>Sportscar</string>
<string>Feline</string>
<string>Soccer</string>
<string>Museum</string>
<string>Baby</string>
<string>Fencing</string>
<string>Railroad</string>
<string>Nascar</string>
<string>Sky Surfing</string>
<string>Bird</string>
<string>Games</string>
<string>Baseball</string>
<string>Dressage</string>
<string>Snorkeling</string>
<string>Pyramid</string>
<string>Kite</string>
<string>Rowboat</string>
<string>Golf</string>
<string>Watersports</string>
<string>Lightning</string>
<string>Canyon</string>
<string>Auditorium</string>
<string>Night Sky</string>
<string>Karaoke</string>
<string>Skiing</string>
<string>Parade</string>
<string>Forest</string>
<string>Hot Air Balloon</string>
<string>Dragon Parade</string>
<string>Easter Egg</string>
<string>Monument</string>
<string>Jungle</string>
<string>Thanksgiving</string>
<string>Jockey Horse</string>
<string>Stadium</string>
<string>Airplane</string>
<string>Ballet</string>
<string>Yoga</string>
<string>Coral Reef</string>
<string>Skating</string>
<string>Wrestling</string>
<string>Bicycle</string>
<string>Tattoo</string>
<string>Amusement Park</string>
<string>Canoe</string>
<string>Cheerleading</string>
<string>Ping Pong</string>
<string>Fishing</string>
<string>Magic</string>
<string>Reptile</string>
<string>Winter Sport</string>
<string>Waterfall</string>
<string>Train</string>
<string>Bonsai</string>
<string>Surfing</string>
<string>Dog</string>
<string>Cake</string>
<string>Sledding</string>
<string>Sandcastle</string>
<string>Glacier</string>
<string>Lighthouse</string>
<string>Equestrian</string>
<string>Rafting</string>
<string>Shore</string>
<string>Hockey</string>
<string>Santa Claus</string>
<string>Formula One Car</string>
<string>Sport</string>
<string>Vehicle</string>
<string>Boxing</string>
<string>Rollerskating</string>
<string>Underwater</string>
<string>Orchestra</string>
<string>Carnival</string>
<string>Rocket</string>
<string>Skateboarding</string>
<string>Helicopter</string>
<string>Performance</string>
<string>Oktoberfest</string>
<string>Water Polo</string>
<string>Skate Park</string>
<string>Animal</string>
<string>Nightclub</string>
<string>String Instrument</string>
<string>Dinosaur</string>
<string>Gymnastics</string>
<string>Cricket</string>
<string>Volcano</string>
<string>Lake</string>
<string>Aurora</string>
<string>Dancing</string>
<string>Concert</string>
<string>Rock Climbing</string>
<string>Hang Glider</string>
<string>Rodeo</string>
<string>Fish</string>
<string>Art</string>
<string>Motorcycle</string>
<string>Volleyball</string>
<string>Wake Boarding</string>
<string>Badminton</string>
<string>Motor Sport</string>
<string>Sumo</string>
<string>Parasailing</string>
<string>Skydiving</string>
<string>Kickboxing</string>
<string>Pinata</string>
<string>Foosball</string>
<string>Go Kart</string>
<string>Poker</string>
<string>Kayak</string>
<string>Swimming</string>
<string>Atv</string>
<string>Beach</string>
<string>Dartboard</string>
<string>Athletics</string>
<string>Camping</string>
<string>Tornado</string>
<string>Billiards</string>
<string>Rugby</string>
<string>Airshow</string>
</array>
</dict>
</plist>

View File

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>insertAlbum</key>
<array/>
<key>insertAsset</key>
<array/>
<key>insertHighlight</key>
<array/>
<key>insertMemory</key>
<array/>
<key>insertMoment</key>
<array/>
<key>removeAlbum</key>
<array/>
<key>removeAsset</key>
<array/>
<key>removeHighlight</key>
<array/>
<key>removeMemory</key>
<array/>
<key>removeMoment</key>
<array/>
</dict>
</plist>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>embeddingVersion</key>
<string>1</string>
<key>localeIdentifier</key>
<string>en_US</string>
<key>sceneTaxonomySHA</key>
<string>87914a047c69fbe8013fad2c70fa70c6c03b08b56190fe4054c880e6b9f57cc3</string>
<key>searchIndexVersion</key>
<string>10</string>
</dict>
</plist>

Binary file not shown.

After

Width:  |  Height:  |  Size: 453 KiB

View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CollapsedSidebarSectionIdentifiers</key>
<array/>
</dict>
</plist>

Some files were not shown because too many files have changed in this diff Show More