forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented retry for export db, #569

Browse Source
This commit is contained in:
Rhet Turnbull
2022-05-24 09:13:44 -07:00
parent 1daf18ad9f
commit dae710b836
2 changed files with 260 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.49.8"
__version__ = "0.49.9"

126
osxphotos/export_db.py
View File

@@ -19,7 +19,7 @@ from sqlite3 import Error
from tempfile import TemporaryDirectory
from typing import Any, List, Optional, Tuple, Union
from tenacity import retry, stop_after_attempt
from tenacity import retry, retry_if_not_exception_type, stop_after_attempt
from ._constants import OSXPHOTOS_EXPORT_DB
from ._version import __version__
@@ -36,7 +36,7 @@ OSXPHOTOS_EXPORTDB_VERSION = "7.1"
OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
# max retry attempts for methods which use tenacity.retry
MAX_RETRY_ATTEMPTS = 5
MAX_RETRY_ATTEMPTS = 3
# maximum number of export results rows to save
MAX_EXPORT_RESULTS_DATA_ROWS = 10
@@ -101,6 +101,7 @@ class ExportDB:
"""returns path to export directory"""
return self._path
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord":
"""get info for filename and uuid
@@ -118,6 +119,10 @@ class ExportDB:
return ExportRecord(conn, filename_normalized)
return None
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_file_record(
self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord":
@@ -136,6 +141,10 @@ class ExportDB:
conn.commit()
return ExportRecord(conn, filename_normalized)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_or_get_file_record(
self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord":
@@ -154,25 +163,22 @@ class ExportDB:
conn.commit()
return ExportRecord(conn, filename_normalized)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_uuid_for_file(self, filename):
"""query database for filename and return UUID
returns None if filename not found in database
"""
filepath_normalized = self._normalize_filepath_relative(filename)
conn = self._conn
try:
c = conn.cursor()
c.execute(
"SELECT uuid FROM export_data WHERE filepath_normalized = ?",
(filepath_normalized,),
)
results = c.fetchone()
uuid = results[0] if results else None
except Error as e:
logging.warning(e)
uuid = None
return uuid
return results[0] if results else None
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_files_for_uuid(self, uuid: str) -> List:
"""query database for UUID and return list of files associated with UUID or empty list"""
conn = self._conn
@@ -184,33 +190,30 @@ class ExportDB:
results = c.fetchall()
return [os.path.join(self.export_dir, r[0]) for r in results]
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_photoinfo_for_uuid(self, uuid):
"""returns the photoinfo JSON struct for a UUID"""
conn = self._conn
try:
c = conn.cursor()
c.execute("SELECT photoinfo FROM photoinfo WHERE uuid = ?", (uuid,))
results = c.fetchone()
info = results[0] if results else None
except Error as e:
logging.warning(e)
info = None
return info
return results[0] if results else None
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_photoinfo_for_uuid(self, uuid, info):
"""sets the photoinfo JSON struct for a UUID"""
conn = self._conn
try:
c = conn.cursor()
c.execute(
"INSERT OR REPLACE INTO photoinfo(uuid, photoinfo) VALUES (?, ?);",
(uuid, info),
)
conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_target_for_file(
self, uuid: str, filename: Union[str, pathlib.Path]
) -> Optional[str]:
@@ -235,28 +238,30 @@ class ExportDB:
for result in results:
filepath_normalized = os.path.splitext(result[2])[0]
if re.match(re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized):
if re.match(
re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized
):
return os.path.join(self.export_dir, result[1])
return None
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_previous_uuids(self):
"""returns list of UUIDs of previously exported photos found in export database"""
conn = self._conn
previous_uuids = []
try:
c = conn.cursor()
c.execute("SELECT DISTINCT uuid FROM export_data")
results = c.fetchall()
previous_uuids = [row[0] for row in results]
except Error as e:
logging.warning(e)
return previous_uuids
return [row[0] for row in results]
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_config(self, config_data):
"""set config in the database"""
conn = self._conn
try:
dt = datetime.datetime.now().isoformat()
c = conn.cursor()
c.execute(
@@ -264,16 +269,17 @@ class ExportDB:
(dt, config_data),
)
conn.commit()
except Error as e:
logging.warning(e)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_export_results(self, results):
"""Store export results in database; data is pickled and gzipped for storage"""
results_data = pickle_and_zip(results)
conn = self._conn
try:
dt = datetime.datetime.now().isoformat()
c = conn.cursor()
c.execute(
@@ -286,9 +292,8 @@ class ExportDB:
(dt, results_data),
)
conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_export_results(self, run: int = 0):
"""Retrieve export results from database
@@ -304,7 +309,6 @@ class ExportDB:
run = -run
conn = self._conn
try:
c = conn.cursor()
c.execute(
"""
@@ -319,32 +323,25 @@ class ExportDB:
results = unzip_and_unpickle(data) if data else None
except IndexError:
results = None
except Error as e:
logging.warning(e)
results = None
return results
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_exported_files(self):
"""Returns tuple of (uuid, filepath) for all paths of all exported files tracked in the database"""
conn = self._conn
try:
c = conn.cursor()
c.execute("SELECT uuid, filepath FROM export_data")
except Error as e:
logging.warning(e)
return
while row := c.fetchone():
yield row[0], os.path.join(self.export_dir, row[1])
return
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self):
"""close the database connection"""
try:
self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile):
"""open export database and return a db connection
if dbfile does not exist, will create and initialize the database
@@ -354,8 +351,6 @@ class ExportDB:
if not os.path.isfile(dbfile):
conn = self._get_db_connection(dbfile)
if not conn:
raise Exception(f"Error getting connection to database {dbfile}")
self._create_or_migrate_db_tables(conn)
self.was_created = True
self.was_upgraded = ()
@@ -379,16 +374,12 @@ class ExportDB:
return conn
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_db_connection(self, dbfile):
"""return db connection to dbname"""
try:
conn = sqlite3.connect(dbfile)
except Error as e:
logging.warning(e)
conn = None
return conn
return sqlite3.connect(dbfile)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_database_version(self, conn):
"""return tuple of (osxphotos, exportdb) versions for database connection conn"""
version_info = conn.execute(
@@ -484,7 +475,6 @@ class ExportDB:
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
]
# create the tables if needed
try:
c = conn.cursor()
for cmd in sql_commands:
c.execute(cmd)
@@ -494,8 +484,6 @@ class ExportDB:
)
c.execute("INSERT INTO about(about) VALUES (?);", (OSXPHOTOS_ABOUT_STRING,))
conn.commit()
except Error as e:
logging.warning(e)
# perform needed migrations
if version[1] < "4.3":
@@ -524,6 +512,7 @@ class ExportDB:
with suppress(Exception):
self._conn.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _insert_run_info(self):
dt = datetime.datetime.now(datetime.timezone.utc).isoformat()
python_path = sys.executable
@@ -531,7 +520,6 @@ class ExportDB:
args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else ""
cwd = os.getcwd()
conn = self._conn
try:
c = conn.cursor()
c.execute(
"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)",
@@ -539,8 +527,6 @@ class ExportDB:
)
conn.commit()
except Error as e:
logging.warning(e)
def _relative_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
"""return filepath relative to self._path"""
@@ -596,16 +582,12 @@ class ExportDB:
def _migrate_4_3_to_5_0(self, conn):
"""Migrate database from version 4.3 to 5.0"""
try:
c = conn.cursor()
# add metadata column to files to support --force-update
c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;")
conn.commit()
except Error as e:
logging.warning(e)
def _migrate_5_0_to_6_0(self, conn):
try:
c = conn.cursor()
# add export_data table
@@ -707,11 +689,8 @@ class ExportDB:
c.execute("DROP TABLE IF EXISTS detected_text;")
conn.commit()
except Error as e:
logging.warning(e)
def _migrate_6_0_to_7_0(self, conn):
try:
c = conn.cursor()
c.execute(
"""CREATE TABLE IF NOT EXISTS export_results_data (
@@ -730,11 +709,8 @@ class ExportDB:
# sleep a tiny bit just to ensure time stamps increment
time.sleep(0.001)
conn.commit()
except Error as e:
logging.warning(e)
def _migrate_7_0_to_7_1(self, conn):
try:
c = conn.cursor()
c.execute("""ALTER TABLE export_data ADD COLUMN timestamp DATETIME;""")
c.execute(
@@ -756,12 +732,9 @@ class ExportDB:
"""
)
conn.commit()
except Error as e:
logging.warning(e)
def _perform_db_maintenace(self, conn):
"""Perform database maintenance"""
try:
c = conn.cursor()
c.execute(
"""DELETE FROM config
@@ -772,8 +745,6 @@ class ExportDB:
"""
)
conn.commit()
except Error as e:
logging.warning(e)
class ExportDBInMemory(ExportDB):
@@ -825,14 +796,13 @@ class ExportDBInMemory(ExportDB):
conn_on_disk.commit()
conn_on_disk.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self):
"""close the database connection"""
try:
if self._conn:
self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile): # sourcery skip: raise-specific-error
"""open export database and return a db connection
returns: connection to the database
@@ -866,13 +836,7 @@ class ExportDBInMemory(ExportDB):
def _get_db_connection(self):
"""return db connection to in memory database"""
try:
conn = sqlite3.connect(":memory:")
except Error as e:
logging.warning(e)
conn = None
return conn
return sqlite3.connect(":memory:")
def _dump_db(self, conn: sqlite3.Connection) -> StringIO:
"""dump sqlite db to a string buffer"""