forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented retry for export db, #569

Browse Source
This commit is contained in:
Rhet Turnbull
2022-05-24 09:13:44 -07:00
parent 1daf18ad9f
commit dae710b836
2 changed files with 260 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """ """ version info """
__version__ = "0.49.8" __version__ = "0.49.9"

126
osxphotos/export_db.py
View File

@@ -19,7 +19,7 @@ from sqlite3 import Error
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from typing import Any, List, Optional, Tuple, Union from typing import Any, List, Optional, Tuple, Union
from tenacity import retry, stop_after_attempt from tenacity import retry, retry_if_not_exception_type, stop_after_attempt
from ._constants import OSXPHOTOS_EXPORT_DB from ._constants import OSXPHOTOS_EXPORT_DB
from ._version import __version__ from ._version import __version__
@@ -36,7 +36,7 @@ OSXPHOTOS_EXPORTDB_VERSION = "7.1"
OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}" OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
# max retry attempts for methods which use tenacity.retry # max retry attempts for methods which use tenacity.retry
MAX_RETRY_ATTEMPTS = 5 MAX_RETRY_ATTEMPTS = 3
# maximum number of export results rows to save # maximum number of export results rows to save
MAX_EXPORT_RESULTS_DATA_ROWS = 10 MAX_EXPORT_RESULTS_DATA_ROWS = 10
@@ -101,6 +101,7 @@ class ExportDB:
"""returns path to export directory""" """returns path to export directory"""
return self._path return self._path
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord": def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord":
"""get info for filename and uuid """get info for filename and uuid
@@ -118,6 +119,10 @@ class ExportDB:
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
return None return None
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_file_record( def create_file_record(
self, filename: Union[pathlib.Path, str], uuid: str self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord": ) -> "ExportRecord":
@@ -136,6 +141,10 @@ class ExportDB:
conn.commit() conn.commit()
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_or_get_file_record( def create_or_get_file_record(
self, filename: Union[pathlib.Path, str], uuid: str self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord": ) -> "ExportRecord":
@@ -154,25 +163,22 @@ class ExportDB:
conn.commit() conn.commit()
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_uuid_for_file(self, filename): def get_uuid_for_file(self, filename):
"""query database for filename and return UUID """query database for filename and return UUID
returns None if filename not found in database returns None if filename not found in database
""" """
filepath_normalized = self._normalize_filepath_relative(filename) filepath_normalized = self._normalize_filepath_relative(filename)
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"SELECT uuid FROM export_data WHERE filepath_normalized = ?", "SELECT uuid FROM export_data WHERE filepath_normalized = ?",
(filepath_normalized,), (filepath_normalized,),
) )
results = c.fetchone() results = c.fetchone()
uuid = results[0] if results else None return results[0] if results else None
except Error as e:
logging.warning(e)
uuid = None
return uuid
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_files_for_uuid(self, uuid: str) -> List: def get_files_for_uuid(self, uuid: str) -> List:
"""query database for UUID and return list of files associated with UUID or empty list""" """query database for UUID and return list of files associated with UUID or empty list"""
conn = self._conn conn = self._conn
@@ -184,33 +190,30 @@ class ExportDB:
results = c.fetchall() results = c.fetchall()
return [os.path.join(self.export_dir, r[0]) for r in results] return [os.path.join(self.export_dir, r[0]) for r in results]
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_photoinfo_for_uuid(self, uuid): def get_photoinfo_for_uuid(self, uuid):
"""returns the photoinfo JSON struct for a UUID""" """returns the photoinfo JSON struct for a UUID"""
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute("SELECT photoinfo FROM photoinfo WHERE uuid = ?", (uuid,)) c.execute("SELECT photoinfo FROM photoinfo WHERE uuid = ?", (uuid,))
results = c.fetchone() results = c.fetchone()
info = results[0] if results else None return results[0] if results else None
except Error as e:
logging.warning(e)
info = None
return info
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_photoinfo_for_uuid(self, uuid, info): def set_photoinfo_for_uuid(self, uuid, info):
"""sets the photoinfo JSON struct for a UUID""" """sets the photoinfo JSON struct for a UUID"""
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"INSERT OR REPLACE INTO photoinfo(uuid, photoinfo) VALUES (?, ?);", "INSERT OR REPLACE INTO photoinfo(uuid, photoinfo) VALUES (?, ?);",
(uuid, info), (uuid, info),
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_target_for_file( def get_target_for_file(
self, uuid: str, filename: Union[str, pathlib.Path] self, uuid: str, filename: Union[str, pathlib.Path]
) -> Optional[str]: ) -> Optional[str]:
@@ -235,28 +238,30 @@ class ExportDB:
for result in results: for result in results:
filepath_normalized = os.path.splitext(result[2])[0] filepath_normalized = os.path.splitext(result[2])[0]
if re.match(re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized): if re.match(
re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized
):
return os.path.join(self.export_dir, result[1]) return os.path.join(self.export_dir, result[1])
return None return None
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_previous_uuids(self): def get_previous_uuids(self):
"""returns list of UUIDs of previously exported photos found in export database""" """returns list of UUIDs of previously exported photos found in export database"""
conn = self._conn conn = self._conn
previous_uuids = [] previous_uuids = []
try:
c = conn.cursor() c = conn.cursor()
c.execute("SELECT DISTINCT uuid FROM export_data") c.execute("SELECT DISTINCT uuid FROM export_data")
results = c.fetchall() results = c.fetchall()
previous_uuids = [row[0] for row in results] return [row[0] for row in results]
except Error as e:
logging.warning(e)
return previous_uuids
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_config(self, config_data): def set_config(self, config_data):
"""set config in the database""" """set config in the database"""
conn = self._conn conn = self._conn
try:
dt = datetime.datetime.now().isoformat() dt = datetime.datetime.now().isoformat()
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
@@ -264,16 +269,17 @@ class ExportDB:
(dt, config_data), (dt, config_data),
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_export_results(self, results): def set_export_results(self, results):
"""Store export results in database; data is pickled and gzipped for storage""" """Store export results in database; data is pickled and gzipped for storage"""
results_data = pickle_and_zip(results) results_data = pickle_and_zip(results)
conn = self._conn conn = self._conn
try:
dt = datetime.datetime.now().isoformat() dt = datetime.datetime.now().isoformat()
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
@@ -286,9 +292,8 @@ class ExportDB:
(dt, results_data), (dt, results_data),
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_export_results(self, run: int = 0): def get_export_results(self, run: int = 0):
"""Retrieve export results from database """Retrieve export results from database
@@ -304,7 +309,6 @@ class ExportDB:
run = -run run = -run
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
""" """
@@ -319,32 +323,25 @@ class ExportDB:
results = unzip_and_unpickle(data) if data else None results = unzip_and_unpickle(data) if data else None
except IndexError: except IndexError:
results = None results = None
except Error as e:
logging.warning(e)
results = None
return results return results
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_exported_files(self): def get_exported_files(self):
"""Returns tuple of (uuid, filepath) for all paths of all exported files tracked in the database""" """Returns tuple of (uuid, filepath) for all paths of all exported files tracked in the database"""
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute("SELECT uuid, filepath FROM export_data") c.execute("SELECT uuid, filepath FROM export_data")
except Error as e:
logging.warning(e)
return
while row := c.fetchone(): while row := c.fetchone():
yield row[0], os.path.join(self.export_dir, row[1]) yield row[0], os.path.join(self.export_dir, row[1])
return return
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self): def close(self):
"""close the database connection""" """close the database connection"""
try:
self._conn.close() self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile): def _open_export_db(self, dbfile):
"""open export database and return a db connection """open export database and return a db connection
if dbfile does not exist, will create and initialize the database if dbfile does not exist, will create and initialize the database
@@ -354,8 +351,6 @@ class ExportDB:
if not os.path.isfile(dbfile): if not os.path.isfile(dbfile):
conn = self._get_db_connection(dbfile) conn = self._get_db_connection(dbfile)
if not conn:
raise Exception(f"Error getting connection to database {dbfile}")
self._create_or_migrate_db_tables(conn) self._create_or_migrate_db_tables(conn)
self.was_created = True self.was_created = True
self.was_upgraded = () self.was_upgraded = ()
@@ -379,16 +374,12 @@ class ExportDB:
return conn return conn
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_db_connection(self, dbfile): def _get_db_connection(self, dbfile):
"""return db connection to dbname""" """return db connection to dbname"""
try: return sqlite3.connect(dbfile)
conn = sqlite3.connect(dbfile)
except Error as e:
logging.warning(e)
conn = None
return conn
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_database_version(self, conn): def _get_database_version(self, conn):
"""return tuple of (osxphotos, exportdb) versions for database connection conn""" """return tuple of (osxphotos, exportdb) versions for database connection conn"""
version_info = conn.execute( version_info = conn.execute(
@@ -484,7 +475,6 @@ class ExportDB:
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""", """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
] ]
# create the tables if needed # create the tables if needed
try:
c = conn.cursor() c = conn.cursor()
for cmd in sql_commands: for cmd in sql_commands:
c.execute(cmd) c.execute(cmd)
@@ -494,8 +484,6 @@ class ExportDB:
) )
c.execute("INSERT INTO about(about) VALUES (?);", (OSXPHOTOS_ABOUT_STRING,)) c.execute("INSERT INTO about(about) VALUES (?);", (OSXPHOTOS_ABOUT_STRING,))
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
# perform needed migrations # perform needed migrations
if version[1] < "4.3": if version[1] < "4.3":
@@ -524,6 +512,7 @@ class ExportDB:
with suppress(Exception): with suppress(Exception):
self._conn.close() self._conn.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _insert_run_info(self): def _insert_run_info(self):
dt = datetime.datetime.now(datetime.timezone.utc).isoformat() dt = datetime.datetime.now(datetime.timezone.utc).isoformat()
python_path = sys.executable python_path = sys.executable
@@ -531,7 +520,6 @@ class ExportDB:
args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "" args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else ""
cwd = os.getcwd() cwd = os.getcwd()
conn = self._conn conn = self._conn
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)", "INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)",
@@ -539,8 +527,6 @@ class ExportDB:
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _relative_filepath(self, filepath: Union[str, pathlib.Path]) -> str: def _relative_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
"""return filepath relative to self._path""" """return filepath relative to self._path"""
@@ -596,16 +582,12 @@ class ExportDB:
def _migrate_4_3_to_5_0(self, conn): def _migrate_4_3_to_5_0(self, conn):
"""Migrate database from version 4.3 to 5.0""" """Migrate database from version 4.3 to 5.0"""
try:
c = conn.cursor() c = conn.cursor()
# add metadata column to files to support --force-update # add metadata column to files to support --force-update
c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;") c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;")
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _migrate_5_0_to_6_0(self, conn): def _migrate_5_0_to_6_0(self, conn):
try:
c = conn.cursor() c = conn.cursor()
# add export_data table # add export_data table
@@ -707,11 +689,8 @@ class ExportDB:
c.execute("DROP TABLE IF EXISTS detected_text;") c.execute("DROP TABLE IF EXISTS detected_text;")
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _migrate_6_0_to_7_0(self, conn): def _migrate_6_0_to_7_0(self, conn):
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"""CREATE TABLE IF NOT EXISTS export_results_data ( """CREATE TABLE IF NOT EXISTS export_results_data (
@@ -730,11 +709,8 @@ class ExportDB:
# sleep a tiny bit just to ensure time stamps increment # sleep a tiny bit just to ensure time stamps increment
time.sleep(0.001) time.sleep(0.001)
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _migrate_7_0_to_7_1(self, conn): def _migrate_7_0_to_7_1(self, conn):
try:
c = conn.cursor() c = conn.cursor()
c.execute("""ALTER TABLE export_data ADD COLUMN timestamp DATETIME;""") c.execute("""ALTER TABLE export_data ADD COLUMN timestamp DATETIME;""")
c.execute( c.execute(
@@ -756,12 +732,9 @@ class ExportDB:
""" """
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _perform_db_maintenace(self, conn): def _perform_db_maintenace(self, conn):
"""Perform database maintenance""" """Perform database maintenance"""
try:
c = conn.cursor() c = conn.cursor()
c.execute( c.execute(
"""DELETE FROM config """DELETE FROM config
@@ -772,8 +745,6 @@ class ExportDB:
""" """
) )
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
class ExportDBInMemory(ExportDB): class ExportDBInMemory(ExportDB):
@@ -825,14 +796,13 @@ class ExportDBInMemory(ExportDB):
conn_on_disk.commit() conn_on_disk.commit()
conn_on_disk.close() conn_on_disk.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self): def close(self):
"""close the database connection""" """close the database connection"""
try:
if self._conn: if self._conn:
self._conn.close() self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile): # sourcery skip: raise-specific-error def _open_export_db(self, dbfile): # sourcery skip: raise-specific-error
"""open export database and return a db connection """open export database and return a db connection
returns: connection to the database returns: connection to the database
@@ -866,13 +836,7 @@ class ExportDBInMemory(ExportDB):
def _get_db_connection(self): def _get_db_connection(self):
"""return db connection to in memory database""" """return db connection to in memory database"""
try: return sqlite3.connect(":memory:")
conn = sqlite3.connect(":memory:")
except Error as e:
logging.warning(e)
conn = None
return conn
def _dump_db(self, conn: sqlite3.Connection) -> StringIO: def _dump_db(self, conn: sqlite3.Connection) -> StringIO:
"""dump sqlite db to a string buffer""" """dump sqlite db to a string buffer"""