forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented retry for export db, #569

Browse Source
This commit is contained in:
Rhet Turnbull
2022-05-24 09:13:44 -07:00
parent 1daf18ad9f
commit dae710b836
2 changed files with 260 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """ """ version info """
__version__ = "0.49.8" __version__ = "0.49.9"

554
osxphotos/export_db.py
View File

@@ -19,7 +19,7 @@ from sqlite3 import Error
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from typing import Any, List, Optional, Tuple, Union from typing import Any, List, Optional, Tuple, Union
from tenacity import retry, stop_after_attempt from tenacity import retry, retry_if_not_exception_type, stop_after_attempt
from ._constants import OSXPHOTOS_EXPORT_DB from ._constants import OSXPHOTOS_EXPORT_DB
from ._version import __version__ from ._version import __version__
@@ -36,7 +36,7 @@ OSXPHOTOS_EXPORTDB_VERSION = "7.1"
OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}" OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}"
# max retry attempts for methods which use tenacity.retry # max retry attempts for methods which use tenacity.retry
MAX_RETRY_ATTEMPTS = 5 MAX_RETRY_ATTEMPTS = 3
# maximum number of export results rows to save # maximum number of export results rows to save
MAX_EXPORT_RESULTS_DATA_ROWS = 10 MAX_EXPORT_RESULTS_DATA_ROWS = 10
@@ -101,6 +101,7 @@ class ExportDB:
"""returns path to export directory""" """returns path to export directory"""
return self._path return self._path
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord": def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord":
"""get info for filename and uuid """get info for filename and uuid
@@ -118,6 +119,10 @@ class ExportDB:
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
return None return None
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_file_record( def create_file_record(
self, filename: Union[pathlib.Path, str], uuid: str self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord": ) -> "ExportRecord":
@@ -136,6 +141,10 @@ class ExportDB:
conn.commit() conn.commit()
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def create_or_get_file_record( def create_or_get_file_record(
self, filename: Union[pathlib.Path, str], uuid: str self, filename: Union[pathlib.Path, str], uuid: str
) -> "ExportRecord": ) -> "ExportRecord":
@@ -154,25 +163,22 @@ class ExportDB:
conn.commit() conn.commit()
return ExportRecord(conn, filename_normalized) return ExportRecord(conn, filename_normalized)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_uuid_for_file(self, filename): def get_uuid_for_file(self, filename):
"""query database for filename and return UUID """query database for filename and return UUID
returns None if filename not found in database returns None if filename not found in database
""" """
filepath_normalized = self._normalize_filepath_relative(filename) filepath_normalized = self._normalize_filepath_relative(filename)
conn = self._conn conn = self._conn
try: c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( "SELECT uuid FROM export_data WHERE filepath_normalized = ?",
"SELECT uuid FROM export_data WHERE filepath_normalized = ?", (filepath_normalized,),
(filepath_normalized,), )
) results = c.fetchone()
results = c.fetchone() return results[0] if results else None
uuid = results[0] if results else None
except Error as e:
logging.warning(e)
uuid = None
return uuid
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_files_for_uuid(self, uuid: str) -> List: def get_files_for_uuid(self, uuid: str) -> List:
"""query database for UUID and return list of files associated with UUID or empty list""" """query database for UUID and return list of files associated with UUID or empty list"""
conn = self._conn conn = self._conn
@@ -184,33 +190,30 @@ class ExportDB:
results = c.fetchall() results = c.fetchall()
return [os.path.join(self.export_dir, r[0]) for r in results] return [os.path.join(self.export_dir, r[0]) for r in results]
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_photoinfo_for_uuid(self, uuid): def get_photoinfo_for_uuid(self, uuid):
"""returns the photoinfo JSON struct for a UUID""" """returns the photoinfo JSON struct for a UUID"""
conn = self._conn conn = self._conn
try: c = conn.cursor()
c = conn.cursor() c.execute("SELECT photoinfo FROM photoinfo WHERE uuid = ?", (uuid,))
c.execute("SELECT photoinfo FROM photoinfo WHERE uuid = ?", (uuid,)) results = c.fetchone()
results = c.fetchone() return results[0] if results else None
info = results[0] if results else None
except Error as e:
logging.warning(e)
info = None
return info
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_photoinfo_for_uuid(self, uuid, info): def set_photoinfo_for_uuid(self, uuid, info):
"""sets the photoinfo JSON struct for a UUID""" """sets the photoinfo JSON struct for a UUID"""
conn = self._conn conn = self._conn
try: c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( "INSERT OR REPLACE INTO photoinfo(uuid, photoinfo) VALUES (?, ?);",
"INSERT OR REPLACE INTO photoinfo(uuid, photoinfo) VALUES (?, ?);", (uuid, info),
(uuid, info), )
) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_target_for_file( def get_target_for_file(
self, uuid: str, filename: Union[str, pathlib.Path] self, uuid: str, filename: Union[str, pathlib.Path]
) -> Optional[str]: ) -> Optional[str]:
@@ -235,60 +238,62 @@ class ExportDB:
for result in results: for result in results:
filepath_normalized = os.path.splitext(result[2])[0] filepath_normalized = os.path.splitext(result[2])[0]
if re.match(re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized): if re.match(
re.escape(filepath_stem) + r"(\s\(\d+\))?$", filepath_normalized
):
return os.path.join(self.export_dir, result[1]) return os.path.join(self.export_dir, result[1])
return None return None
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_previous_uuids(self): def get_previous_uuids(self):
"""returns list of UUIDs of previously exported photos found in export database""" """returns list of UUIDs of previously exported photos found in export database"""
conn = self._conn conn = self._conn
previous_uuids = [] previous_uuids = []
try: c = conn.cursor()
c = conn.cursor() c.execute("SELECT DISTINCT uuid FROM export_data")
c.execute("SELECT DISTINCT uuid FROM export_data") results = c.fetchall()
results = c.fetchall() return [row[0] for row in results]
previous_uuids = [row[0] for row in results]
except Error as e:
logging.warning(e)
return previous_uuids
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_config(self, config_data): def set_config(self, config_data):
"""set config in the database""" """set config in the database"""
conn = self._conn conn = self._conn
try: dt = datetime.datetime.now().isoformat()
dt = datetime.datetime.now().isoformat() c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( "INSERT OR REPLACE INTO config(datetime, config) VALUES (?, ?);",
"INSERT OR REPLACE INTO config(datetime, config) VALUES (?, ?);", (dt, config_data),
(dt, config_data), )
) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
@retry(
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
retry=retry_if_not_exception_type(sqlite3.IntegrityError),
)
def set_export_results(self, results): def set_export_results(self, results):
"""Store export results in database; data is pickled and gzipped for storage""" """Store export results in database; data is pickled and gzipped for storage"""
results_data = pickle_and_zip(results) results_data = pickle_and_zip(results)
conn = self._conn conn = self._conn
try: dt = datetime.datetime.now().isoformat()
dt = datetime.datetime.now().isoformat() c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( """
""" UPDATE export_results_data
UPDATE export_results_data SET datetime = ?,
SET datetime = ?, export_results = ?
export_results = ? WHERE datetime = (SELECT MIN(datetime) FROM export_results_data);
WHERE datetime = (SELECT MIN(datetime) FROM export_results_data); """,
""", (dt, results_data),
(dt, results_data), )
) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_export_results(self, run: int = 0): def get_export_results(self, run: int = 0):
"""Retrieve export results from database """Retrieve export results from database
@@ -304,47 +309,39 @@ class ExportDB:
run = -run run = -run
conn = self._conn conn = self._conn
c = conn.cursor()
c.execute(
"""
SELECT export_results
FROM export_results_data
ORDER BY datetime DESC
""",
)
rows = c.fetchall()
try: try:
c = conn.cursor() data = rows[run][0]
c.execute( results = unzip_and_unpickle(data) if data else None
""" except IndexError:
SELECT export_results
FROM export_results_data
ORDER BY datetime DESC
""",
)
rows = c.fetchall()
try:
data = rows[run][0]
results = unzip_and_unpickle(data) if data else None
except IndexError:
results = None
except Error as e:
logging.warning(e)
results = None results = None
return results return results
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def get_exported_files(self): def get_exported_files(self):
"""Returns tuple of (uuid, filepath) for all paths of all exported files tracked in the database""" """Returns tuple of (uuid, filepath) for all paths of all exported files tracked in the database"""
conn = self._conn conn = self._conn
try: c = conn.cursor()
c = conn.cursor() c.execute("SELECT uuid, filepath FROM export_data")
c.execute("SELECT uuid, filepath FROM export_data")
except Error as e:
logging.warning(e)
return
while row := c.fetchone(): while row := c.fetchone():
yield row[0], os.path.join(self.export_dir, row[1]) yield row[0], os.path.join(self.export_dir, row[1])
return return
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self): def close(self):
"""close the database connection""" """close the database connection"""
try: self._conn.close()
self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile): def _open_export_db(self, dbfile):
"""open export database and return a db connection """open export database and return a db connection
if dbfile does not exist, will create and initialize the database if dbfile does not exist, will create and initialize the database
@@ -354,8 +351,6 @@ class ExportDB:
if not os.path.isfile(dbfile): if not os.path.isfile(dbfile):
conn = self._get_db_connection(dbfile) conn = self._get_db_connection(dbfile)
if not conn:
raise Exception(f"Error getting connection to database {dbfile}")
self._create_or_migrate_db_tables(conn) self._create_or_migrate_db_tables(conn)
self.was_created = True self.was_created = True
self.was_upgraded = () self.was_upgraded = ()
@@ -379,16 +374,12 @@ class ExportDB:
return conn return conn
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_db_connection(self, dbfile): def _get_db_connection(self, dbfile):
"""return db connection to dbname""" """return db connection to dbname"""
try: return sqlite3.connect(dbfile)
conn = sqlite3.connect(dbfile)
except Error as e:
logging.warning(e)
conn = None
return conn
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _get_database_version(self, conn): def _get_database_version(self, conn):
"""return tuple of (osxphotos, exportdb) versions for database connection conn""" """return tuple of (osxphotos, exportdb) versions for database connection conn"""
version_info = conn.execute( version_info = conn.execute(
@@ -484,18 +475,15 @@ class ExportDB:
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""", """ CREATE UNIQUE INDEX IF NOT EXISTS idx_detected_text on detected_text (uuid);""",
] ]
# create the tables if needed # create the tables if needed
try: c = conn.cursor()
c = conn.cursor() for cmd in sql_commands:
for cmd in sql_commands: c.execute(cmd)
c.execute(cmd) c.execute(
c.execute( "INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);",
"INSERT INTO version(osxphotos, exportdb) VALUES (?, ?);", (__version__, OSXPHOTOS_EXPORTDB_VERSION),
(__version__, OSXPHOTOS_EXPORTDB_VERSION), )
) c.execute("INSERT INTO about(about) VALUES (?);", (OSXPHOTOS_ABOUT_STRING,))
c.execute("INSERT INTO about(about) VALUES (?);", (OSXPHOTOS_ABOUT_STRING,)) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
# perform needed migrations # perform needed migrations
if version[1] < "4.3": if version[1] < "4.3":
@@ -524,6 +512,7 @@ class ExportDB:
with suppress(Exception): with suppress(Exception):
self._conn.close() self._conn.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _insert_run_info(self): def _insert_run_info(self):
dt = datetime.datetime.now(datetime.timezone.utc).isoformat() dt = datetime.datetime.now(datetime.timezone.utc).isoformat()
python_path = sys.executable python_path = sys.executable
@@ -531,16 +520,13 @@ class ExportDB:
args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "" args = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else ""
cwd = os.getcwd() cwd = os.getcwd()
conn = self._conn conn = self._conn
try: c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( "INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)",
"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)", (dt, python_path, cmd, args, cwd),
(dt, python_path, cmd, args, cwd), )
)
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _relative_filepath(self, filepath: Union[str, pathlib.Path]) -> str: def _relative_filepath(self, filepath: Union[str, pathlib.Path]) -> str:
"""return filepath relative to self._path""" """return filepath relative to self._path"""
@@ -596,184 +582,169 @@ class ExportDB:
def _migrate_4_3_to_5_0(self, conn): def _migrate_4_3_to_5_0(self, conn):
"""Migrate database from version 4.3 to 5.0""" """Migrate database from version 4.3 to 5.0"""
try: c = conn.cursor()
c = conn.cursor() # add metadata column to files to support --force-update
# add metadata column to files to support --force-update c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;")
c.execute("ALTER TABLE files ADD COLUMN metadata TEXT;") conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
def _migrate_5_0_to_6_0(self, conn): def _migrate_5_0_to_6_0(self, conn):
try: c = conn.cursor()
c = conn.cursor()
# add export_data table # add export_data table
c.execute( c.execute(
""" CREATE TABLE IF NOT EXISTS export_data( """ CREATE TABLE IF NOT EXISTS export_data(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
filepath_normalized TEXT NOT NULL, filepath_normalized TEXT NOT NULL,
filepath TEXT NOT NULL, filepath TEXT NOT NULL,
uuid TEXT NOT NULL, uuid TEXT NOT NULL,
src_mode INTEGER, src_mode INTEGER,
src_size INTEGER, src_size INTEGER,
src_mtime REAL, src_mtime REAL,
dest_mode INTEGER, dest_mode INTEGER,
dest_size INTEGER, dest_size INTEGER,
dest_mtime REAL, dest_mtime REAL,
digest TEXT, digest TEXT,
exifdata JSON, exifdata JSON,
export_options INTEGER, export_options INTEGER,
UNIQUE(filepath_normalized) UNIQUE(filepath_normalized)
); """, ); """,
) )
c.execute( c.execute(
""" CREATE UNIQUE INDEX IF NOT EXISTS idx_export_data_filepath_normalized on export_data (filepath_normalized); """, """ CREATE UNIQUE INDEX IF NOT EXISTS idx_export_data_filepath_normalized on export_data (filepath_normalized); """,
) )
# migrate data # migrate data
c.execute( c.execute(
""" INSERT INTO export_data (filepath_normalized, filepath, uuid) SELECT filepath_normalized, filepath, uuid FROM files;""", """ INSERT INTO export_data (filepath_normalized, filepath, uuid) SELECT filepath_normalized, filepath, uuid FROM files;""",
) )
c.execute( c.execute(
""" UPDATE export_data """ UPDATE export_data
SET (src_mode, src_size, src_mtime) = SET (src_mode, src_size, src_mtime) =
(SELECT mode, size, mtime (SELECT mode, size, mtime
FROM edited FROM edited
WHERE export_data.filepath_normalized = edited.filepath_normalized); WHERE export_data.filepath_normalized = edited.filepath_normalized);
""", """,
) )
c.execute( c.execute(
""" UPDATE export_data """ UPDATE export_data
SET (dest_mode, dest_size, dest_mtime) = SET (dest_mode, dest_size, dest_mtime) =
(SELECT orig_mode, orig_size, orig_mtime (SELECT orig_mode, orig_size, orig_mtime
FROM files FROM files
WHERE export_data.filepath_normalized = files.filepath_normalized); WHERE export_data.filepath_normalized = files.filepath_normalized);
""", """,
) )
c.execute( c.execute(
""" UPDATE export_data SET digest = """ UPDATE export_data SET digest =
(SELECT metadata FROM files (SELECT metadata FROM files
WHERE files.filepath_normalized = export_data.filepath_normalized WHERE files.filepath_normalized = export_data.filepath_normalized
); """ ); """
) )
c.execute( c.execute(
""" UPDATE export_data SET exifdata = """ UPDATE export_data SET exifdata =
(SELECT json_exifdata FROM exifdata (SELECT json_exifdata FROM exifdata
WHERE exifdata.filepath_normalized = export_data.filepath_normalized WHERE exifdata.filepath_normalized = export_data.filepath_normalized
); """ ); """
) )
# create config table # create config table
c.execute( c.execute(
""" CREATE TABLE IF NOT EXISTS config ( """ CREATE TABLE IF NOT EXISTS config (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
datetime TEXT, datetime TEXT,
config TEXT config TEXT
); """ ); """
) )
# create photoinfo table # create photoinfo table
c.execute( c.execute(
""" CREATE TABLE IF NOT EXISTS photoinfo ( """ CREATE TABLE IF NOT EXISTS photoinfo (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL, uuid TEXT NOT NULL,
photoinfo JSON, photoinfo JSON,
UNIQUE(uuid) UNIQUE(uuid)
); """ ); """
) )
c.execute( c.execute(
"""CREATE UNIQUE INDEX IF NOT EXISTS idx_photoinfo_uuid on photoinfo (uuid);""" """CREATE UNIQUE INDEX IF NOT EXISTS idx_photoinfo_uuid on photoinfo (uuid);"""
) )
c.execute( c.execute(
""" INSERT INTO photoinfo (uuid, photoinfo) SELECT uuid, json_info FROM info;""" """ INSERT INTO photoinfo (uuid, photoinfo) SELECT uuid, json_info FROM info;"""
) )
# drop indexes no longer needed # drop indexes no longer needed
c.execute("DROP INDEX IF EXISTS idx_files_filepath_normalized;") c.execute("DROP INDEX IF EXISTS idx_files_filepath_normalized;")
c.execute("DROP INDEX IF EXISTS idx_exifdata_filename;") c.execute("DROP INDEX IF EXISTS idx_exifdata_filename;")
c.execute("DROP INDEX IF EXISTS idx_edited_filename;") c.execute("DROP INDEX IF EXISTS idx_edited_filename;")
c.execute("DROP INDEX IF EXISTS idx_converted_filename;") c.execute("DROP INDEX IF EXISTS idx_converted_filename;")
c.execute("DROP INDEX IF EXISTS idx_sidecar_filename;") c.execute("DROP INDEX IF EXISTS idx_sidecar_filename;")
c.execute("DROP INDEX IF EXISTS idx_detected_text;") c.execute("DROP INDEX IF EXISTS idx_detected_text;")
# drop tables no longer needed # drop tables no longer needed
c.execute("DROP TABLE IF EXISTS files;") c.execute("DROP TABLE IF EXISTS files;")
c.execute("DROP TABLE IF EXISTS info;") c.execute("DROP TABLE IF EXISTS info;")
c.execute("DROP TABLE IF EXISTS exifdata;") c.execute("DROP TABLE IF EXISTS exifdata;")
c.execute("DROP TABLE IF EXISTS edited;") c.execute("DROP TABLE IF EXISTS edited;")
c.execute("DROP TABLE IF EXISTS converted;") c.execute("DROP TABLE IF EXISTS converted;")
c.execute("DROP TABLE IF EXISTS sidecar;") c.execute("DROP TABLE IF EXISTS sidecar;")
c.execute("DROP TABLE IF EXISTS detected_text;") c.execute("DROP TABLE IF EXISTS detected_text;")
conn.commit() conn.commit()
except Error as e:
logging.warning(e)
def _migrate_6_0_to_7_0(self, conn): def _migrate_6_0_to_7_0(self, conn):
try: c = conn.cursor()
c = conn.cursor() c.execute(
"""CREATE TABLE IF NOT EXISTS export_results_data (
id INTEGER PRIMARY KEY,
datetime TEXT,
export_results BLOB
);"""
)
# pre-populate report_data table with blank fields
# ExportDB will use these as circular buffer always writing to the oldest record
for _ in range(MAX_EXPORT_RESULTS_DATA_ROWS):
c.execute( c.execute(
"""CREATE TABLE IF NOT EXISTS export_results_data ( """INSERT INTO export_results_data (datetime, export_results) VALUES (?, ?);""",
id INTEGER PRIMARY KEY, (datetime.datetime.now().isoformat(), b""),
datetime TEXT,
export_results BLOB
);"""
) )
# pre-populate report_data table with blank fields # sleep a tiny bit just to ensure time stamps increment
# ExportDB will use these as circular buffer always writing to the oldest record time.sleep(0.001)
for _ in range(MAX_EXPORT_RESULTS_DATA_ROWS): conn.commit()
c.execute(
"""INSERT INTO export_results_data (datetime, export_results) VALUES (?, ?);""",
(datetime.datetime.now().isoformat(), b""),
)
# sleep a tiny bit just to ensure time stamps increment
time.sleep(0.001)
conn.commit()
except Error as e:
logging.warning(e)
def _migrate_7_0_to_7_1(self, conn): def _migrate_7_0_to_7_1(self, conn):
try: c = conn.cursor()
c = conn.cursor() c.execute("""ALTER TABLE export_data ADD COLUMN timestamp DATETIME;""")
c.execute("""ALTER TABLE export_data ADD COLUMN timestamp DATETIME;""") c.execute(
c.execute( """
""" CREATE TRIGGER insert_timestamp_trigger
CREATE TRIGGER insert_timestamp_trigger AFTER INSERT ON export_data
AFTER INSERT ON export_data BEGIN
BEGIN UPDATE export_data SET timestamp = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE id = NEW.id;
UPDATE export_data SET timestamp = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE id = NEW.id; END;
END; """
""" )
) c.execute(
c.execute( """
""" CREATE TRIGGER update_timestamp_trigger
CREATE TRIGGER update_timestamp_trigger AFTER UPDATE On export_data
AFTER UPDATE On export_data BEGIN
BEGIN UPDATE export_data SET timestamp = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE id = NEW.id;
UPDATE export_data SET timestamp = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE id = NEW.id; END;
END; """
""" )
) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
def _perform_db_maintenace(self, conn): def _perform_db_maintenace(self, conn):
"""Perform database maintenance""" """Perform database maintenance"""
try: c = conn.cursor()
c = conn.cursor() c.execute(
c.execute( """DELETE FROM config
"""DELETE FROM config WHERE id < (
WHERE id < ( SELECT MIN(id)
SELECT MIN(id) FROM (SELECT id FROM config ORDER BY id DESC LIMIT 9)
FROM (SELECT id FROM config ORDER BY id DESC LIMIT 9) );
); """
""" )
) conn.commit()
conn.commit()
except Error as e:
logging.warning(e)
class ExportDBInMemory(ExportDB): class ExportDBInMemory(ExportDB):
@@ -825,14 +796,13 @@ class ExportDBInMemory(ExportDB):
conn_on_disk.commit() conn_on_disk.commit()
conn_on_disk.close() conn_on_disk.close()
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def close(self): def close(self):
"""close the database connection""" """close the database connection"""
try: if self._conn:
if self._conn: self._conn.close()
self._conn.close()
except Error as e:
logging.warning(e)
@retry(stop=stop_after_attempt(MAX_RETRY_ATTEMPTS))
def _open_export_db(self, dbfile): # sourcery skip: raise-specific-error def _open_export_db(self, dbfile): # sourcery skip: raise-specific-error
"""open export database and return a db connection """open export database and return a db connection
returns: connection to the database returns: connection to the database
@@ -866,13 +836,7 @@ class ExportDBInMemory(ExportDB):
def _get_db_connection(self): def _get_db_connection(self):
"""return db connection to in memory database""" """return db connection to in memory database"""
try: return sqlite3.connect(":memory:")
conn = sqlite3.connect(":memory:")
except Error as e:
logging.warning(e)
conn = None
return conn
def _dump_db(self, conn: sqlite3.Connection) -> StringIO: def _dump_db(self, conn: sqlite3.Connection) -> StringIO:
"""dump sqlite db to a string buffer""" """dump sqlite db to a string buffer"""