forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added --ramdb option (#639)

Browse Source
This commit is contained in:
Rhet Turnbull
2022-02-21 12:20:02 -07:00
committed by GitHub
parent 1941e79d21
commit b92a681795
18 changed files with 280 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.46.0"
__version__ = "0.46.1"

28
osxphotos/cli.py
View File

@@ -1190,6 +1190,13 @@ def cli(ctx, db, json_, debug):
),
type=ExportDBType(),
)
@click.option(
"--ramdb",
is_flag=True,
help="Copy export database to memory during export; "
"may improve performance when exporting over a network or slow disk but could result in "
"losing update state information if the program is interrupted or crashes.",
)
@click.option(
"--load-config",
required=False,
@@ -1383,6 +1390,7 @@ def export(
add_skipped_to_album,
add_missing_to_album,
exportdb,
ramdb,
load_config,
save_config,
config_only,
@@ -1501,6 +1509,7 @@ def export(
export_as_hardlink = cfg.export_as_hardlink
export_by_date = cfg.export_by_date
exportdb = cfg.exportdb
ramdb = cfg.ramdb
external_edit = cfg.external_edit
favorite = cfg.favorite
filename_template = cfg.filename_template
@@ -1802,7 +1811,7 @@ def export(
)
)
# open export database and assign copy/link/unlink functions
# open export database
export_db_path = exportdb or os.path.join(dest, OSXPHOTOS_EXPORT_DB)
# check that export isn't in the parent or child of a previously exported library
@@ -1829,7 +1838,11 @@ def export(
export_db = ExportDBInMemory(dbfile=export_db_path, export_dir=dest)
fileutil = FileUtilNoOp
else:
export_db = ExportDB(dbfile=export_db_path, export_dir=dest)
export_db = (
ExportDBInMemory(dbfile=export_db_path, export_dir=dest)
if ramdb
else ExportDB(dbfile=export_db_path, export_dir=dest)
)
fileutil = FileUtil
if verbose_:
@@ -2212,6 +2225,10 @@ def export(
verbose_(f"Writing export report to {report}")
write_export_report(report, results)
# close export_db and write changes if needed
if ramdb and not dry_run:
verbose_(f"Writing export database changes back to {export_db.path}")
export_db.write_to_disk()
export_db.close()
@@ -4789,7 +4806,7 @@ def run(python_file):
@click.option(
"--migrate",
is_flag=True,
help="Migrate (if needed) export database to current version."
help="Migrate (if needed) export database to current version.",
)
@click.option(
"--export-dir",
@@ -4953,9 +4970,12 @@ def exportdb(
f"Migrated export database {export_db} from version {upgraded[0]} to {upgraded[1]}"
)
else:
print(f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}")
print(
f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}"
)
sys.exit(0)
def _query_options_from_kwargs(**kwargs) -> QueryOptions:
"""Validate query options and create a QueryOptions instance"""
# sanity check input args

83
osxphotos/export_db.py
View File

@@ -50,6 +50,16 @@ class ExportDB:
self._perform_db_maintenace(self._conn)
self._insert_run_info()
@property
def path(self):
"""returns path to export database"""
return self._dbfile
@property
def export_dir(self):
"""returns path to export directory"""
return self._path
def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord":
"""get info for filename and uuid
@@ -566,7 +576,14 @@ class ExportDBInMemory(ExportDB):
modifying the on-disk version
"""
def __init__(self, dbfile, export_dir):
def __init__(self, dbfile: str, export_dir: str):
""" "Initialize ExportDBInMemory
Args:
dbfile (str): path to database file
export_dir (str): path to export directory
write_back (bool): whether to write changes back to disk when closing; if False (default), changes are not written to disk
"""
self._dbfile = dbfile or f"./{OSXPHOTOS_EXPORT_DB}"
# export_dir is required as all files referenced by get_/set_uuid_for_file will be converted to
# relative paths to this path
@@ -576,6 +593,39 @@ class ExportDBInMemory(ExportDB):
self._conn = self._open_export_db(self._dbfile)
self._insert_run_info()
def write_to_disk(self):
"""Write changes from in-memory database back to disk"""
# dump the database
conn = self._conn
conn.commit()
dbdump = self._dump_db(conn)
# cleanup the old on-disk database
# also unlink the wal and shm files if needed
dbfile = pathlib.Path(self._dbfile)
if dbfile.exists():
dbfile.unlink()
wal = dbfile.with_suffix(".db-wal")
if wal.exists():
wal.unlink()
shm = dbfile.with_suffix(".db-shm")
if shm.exists():
shm.unlink()
conn_on_disk = sqlite3.connect(str(dbfile))
conn_on_disk.cursor().executescript(dbdump.read())
conn_on_disk.commit()
conn_on_disk.close()
def close(self):
"""close the database connection"""
try:
if self._conn:
self._conn.close()
except Error as e:
logging.warning(e)
def _open_export_db(self, dbfile):
"""open export database and return a db connection
returns: connection to the database
@@ -588,21 +638,13 @@ class ExportDBInMemory(ExportDB):
self.was_created = True
self.was_upgraded = ()
else:
try:
conn = sqlite3.connect(dbfile)
except Error as e:
logging.warning(e)
raise e from e
tempfile = StringIO()
for line in conn.iterdump():
tempfile.write("%s\n" % line)
conn = sqlite3.connect(dbfile)
dbdump = self._dump_db(conn)
conn.close()
tempfile.seek(0)
# Create a database in memory and import from tempfile
# Create a database in memory and import from the dump
conn = sqlite3.connect(":memory:")
conn.cursor().executescript(tempfile.read())
conn.cursor().executescript(dbdump.read())
conn.commit()
self.was_created = False
version_info = self._get_database_version(conn)
@@ -625,6 +667,21 @@ class ExportDBInMemory(ExportDB):
return conn
def _dump_db(self, conn: sqlite3.Connection) -> StringIO:
"""dump sqlite db to a string buffer"""
dbdump = StringIO()
for line in conn.iterdump():
dbdump.write("%s\n" % line)
dbdump.seek(0)
return dbdump
def __del__(self):
"""close the database connection"""
try:
self.close()
except Error as e:
pass
class ExportDBTemp(ExportDBInMemory):
"""Temporary in-memory version of ExportDB"""