forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added --ramdb option (#639)

Browse Source
This commit is contained in:
Rhet Turnbull
2022-02-21 12:20:02 -07:00
committed by GitHub
parent 1941e79d21
commit b92a681795
18 changed files with 280 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -1173,6 +1173,11 @@ Options:
'.osxphotos_export.db' in the export
directory. If --exportdb is specified, it
will be saved to the specified file.
--ramdb Copy export database to memory during export;
may improve performance when exporting over a
network or slow disk but could result in
losing update state information if the program
is interrupted or crashes.
--load-config <config file path>
Load options from file as written with --save-
config. This allows you to save a complex
@@ -1736,7 +1741,7 @@ Substitution Description
{lf} A line feed: '\n', alias for {newline}
{cr} A carriage return: '\r'
{crlf} a carriage return + line feed: '\r\n'
{osxphotos_version} The osxphotos version, e.g. '0.46.0'
{osxphotos_version} The osxphotos version, e.g. '0.46.1'
{osxphotos_cmd_line} The full command line used to run osxphotos
The following substitutions may result in multiple values. Thus if specified for
@@ -3640,7 +3645,7 @@ The following template field substitutions are availabe for use the templating s
|{lf}|A line feed: '\n', alias for {newline}|
|{cr}|A carriage return: '\r'|
|{crlf}|a carriage return + line feed: '\r\n'|
|{osxphotos_version}|The osxphotos version, e.g. '0.46.0'|
|{osxphotos_version}|The osxphotos version, e.g. '0.46.1'|
|{osxphotos_cmd_line}|The full command line used to run osxphotos|
|{album}|Album(s) photo is contained in|
|{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|

2
docs/.buildinfo
View File

@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 5b6236594d7900f08d9a1afda487bf3c
config: d6da9902a4771e5081ae73c361960af8
tags: 645f666f9bcd5a90fca523b33c5a78b7

2
docs/_modules/index.html
View File

@@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Overview: module code &#8212; osxphotos 0.46.0 documentation</title>
<title>Overview: module code &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../_static/alabaster.css" />
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>

3
docs/_modules/osxphotos/photoinfo.html
View File

@@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>osxphotos.photoinfo &#8212; osxphotos 0.46.0 documentation</title>
<title>osxphotos.photoinfo &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/alabaster.css" />
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
@@ -1818,7 +1818,6 @@
<h3>Navigation</h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../cli.html">osxphotos command line interface (CLI)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules.html">osxphotos</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../reference.html">osxphotos package</a></li>
</ul>

2
docs/_static/documentation_options.js vendored
View File

@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '0.46.0',
VERSION: '0.46.1',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
BUILDER: 'html',

8
docs/cli.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos command line interface (CLI) &#8212; osxphotos 0.46.0 documentation</title>
<title>osxphotos command line interface (CLI) &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
@@ -986,6 +986,12 @@ to modify this behavior.</p>
<dd><p>Specify alternate path for database file which stores state information for export and –update. If –exportdb is not specified, export database will be saved to ‘.osxphotos_export.db’ in the export directory. If –exportdb is specified, it will be saved to the specified file.</p>
</dd></dl>
<dl class="std option">
<dt class="sig sig-object std" id="cmdoption-osxphotos-export-ramdb">
<span class="sig-name descname"><span class="pre">--ramdb</span></span><span class="sig-prename descclassname"></span><a class="headerlink" href="#cmdoption-osxphotos-export-ramdb" title="Permalink to this definition"></a></dt>
<dd><p>Copy export database to memory during export; may improve performance when exporting over a network or slow disk but could result in losing update state information if the program is interrupted or crashes.</p>
</dd></dl>
<dl class="std option">
<dt class="sig sig-object std" id="cmdoption-osxphotos-export-load-config">
<span class="sig-name descname"><span class="pre">--load-config</span></span><span class="sig-prename descclassname"> <span class="pre">&lt;config</span> <span class="pre">file</span> <span class="pre">path&gt;</span></span><a class="headerlink" href="#cmdoption-osxphotos-export-load-config" title="Permalink to this definition"></a></dt>

11
docs/genindex.html
View File

@@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Index &#8212; osxphotos 0.46.0 documentation</title>
<title>Index &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
@@ -1084,6 +1084,13 @@
<li><a href="cli.html#cmdoption-osxphotos-query-query-function">osxphotos-query command line option</a>
</li>
<li><a href="cli.html#cmdoption-osxphotos-repl-query-function">osxphotos-repl command line option</a>
</li>
</ul></li>
<li>
--ramdb
<ul>
<li><a href="cli.html#cmdoption-osxphotos-export-ramdb">osxphotos-export command line option</a>
</li>
</ul></li>
<li>
@@ -2012,6 +2019,8 @@
<li><a href="cli.html#cmdoption-osxphotos-export-query-eval">--query-eval &lt;CRITERIA&gt;</a>
</li>
<li><a href="cli.html#cmdoption-osxphotos-export-query-function">--query-function &lt;filename.py::function&gt;</a>
</li>
<li><a href="cli.html#cmdoption-osxphotos-export-ramdb">--ramdb</a>
</li>
<li><a href="cli.html#cmdoption-osxphotos-export-regex">--regex &lt;REGEX TEMPLATE&gt;</a>
</li>

2
docs/index.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.46.0 documentation</title>
<title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/modules.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos &#8212; osxphotos 0.46.0 documentation</title>
<title>osxphotos &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

BIN
docs/objects.inv
View File

Binary file not shown.

7
docs/reference.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos package &#8212; osxphotos 0.46.0 documentation</title>
<title>osxphotos package &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
@@ -15,7 +15,7 @@
<script src="_static/doctools.js"></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="prev" title="osxphotos" href="modules.html" />
<link rel="prev" title="osxphotos command line interface (CLI)" href="cli.html" />
<link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -935,7 +935,6 @@ Returns None if no associated RAW image</p>
<h3>Navigation</h3>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="cli.html">osxphotos command line interface (CLI)</a></li>
<li class="toctree-l1"><a class="reference internal" href="modules.html">osxphotos</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">osxphotos package</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#osxphotos-module">osxphotos module</a></li>
</ul>
@@ -946,7 +945,7 @@ Returns None if no associated RAW image</p>
<h3>Related Topics</h3>
<ul>
<li><a href="index.html">Documentation overview</a><ul>
<li>Previous: <a href="modules.html" title="previous chapter">osxphotos</a></li>
<li>Previous: <a href="cli.html" title="previous chapter">osxphotos command line interface (CLI)</a></li>
</ul></li>
</ul>
</div>

2
docs/search.html
View File

@@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Search &#8212; osxphotos 0.46.0 documentation</title>
<title>Search &#8212; osxphotos 0.46.1 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />

2
docs/searchindex.js
View File

File diff suppressed because one or more lines are too long

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.46.0"
__version__ = "0.46.1"

28
osxphotos/cli.py
View File

@@ -1190,6 +1190,13 @@ def cli(ctx, db, json_, debug):
),
type=ExportDBType(),
)
@click.option(
"--ramdb",
is_flag=True,
help="Copy export database to memory during export; "
"may improve performance when exporting over a network or slow disk but could result in "
"losing update state information if the program is interrupted or crashes.",
)
@click.option(
"--load-config",
required=False,
@@ -1383,6 +1390,7 @@ def export(
add_skipped_to_album,
add_missing_to_album,
exportdb,
ramdb,
load_config,
save_config,
config_only,
@@ -1501,6 +1509,7 @@ def export(
export_as_hardlink = cfg.export_as_hardlink
export_by_date = cfg.export_by_date
exportdb = cfg.exportdb
ramdb = cfg.ramdb
external_edit = cfg.external_edit
favorite = cfg.favorite
filename_template = cfg.filename_template
@@ -1802,7 +1811,7 @@ def export(
)
)
# open export database and assign copy/link/unlink functions
# open export database
export_db_path = exportdb or os.path.join(dest, OSXPHOTOS_EXPORT_DB)
# check that export isn't in the parent or child of a previously exported library
@@ -1829,7 +1838,11 @@ def export(
export_db = ExportDBInMemory(dbfile=export_db_path, export_dir=dest)
fileutil = FileUtilNoOp
else:
export_db = ExportDB(dbfile=export_db_path, export_dir=dest)
export_db = (
ExportDBInMemory(dbfile=export_db_path, export_dir=dest)
if ramdb
else ExportDB(dbfile=export_db_path, export_dir=dest)
)
fileutil = FileUtil
if verbose_:
@@ -2212,6 +2225,10 @@ def export(
verbose_(f"Writing export report to {report}")
write_export_report(report, results)
# close export_db and write changes if needed
if ramdb and not dry_run:
verbose_(f"Writing export database changes back to {export_db.path}")
export_db.write_to_disk()
export_db.close()
@@ -4789,7 +4806,7 @@ def run(python_file):
@click.option(
"--migrate",
is_flag=True,
help="Migrate (if needed) export database to current version."
help="Migrate (if needed) export database to current version.",
)
@click.option(
"--export-dir",
@@ -4953,9 +4970,12 @@ def exportdb(
f"Migrated export database {export_db} from version {upgraded[0]} to {upgraded[1]}"
)
else:
print(f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}")
print(
f"Export database {export_db} is already at latest version {OSXPHOTOS_EXPORTDB_VERSION}"
)
sys.exit(0)
def _query_options_from_kwargs(**kwargs) -> QueryOptions:
"""Validate query options and create a QueryOptions instance"""
# sanity check input args

83
osxphotos/export_db.py
View File

@@ -50,6 +50,16 @@ class ExportDB:
self._perform_db_maintenace(self._conn)
self._insert_run_info()
@property
def path(self):
"""returns path to export database"""
return self._dbfile
@property
def export_dir(self):
"""returns path to export directory"""
return self._path
def get_file_record(self, filename: Union[pathlib.Path, str]) -> "ExportRecord":
"""get info for filename and uuid
@@ -566,7 +576,14 @@ class ExportDBInMemory(ExportDB):
modifying the on-disk version
"""
def __init__(self, dbfile, export_dir):
def __init__(self, dbfile: str, export_dir: str):
""" "Initialize ExportDBInMemory
Args:
dbfile (str): path to database file
export_dir (str): path to export directory
write_back (bool): whether to write changes back to disk when closing; if False (default), changes are not written to disk
"""
self._dbfile = dbfile or f"./{OSXPHOTOS_EXPORT_DB}"
# export_dir is required as all files referenced by get_/set_uuid_for_file will be converted to
# relative paths to this path
@@ -576,6 +593,39 @@ class ExportDBInMemory(ExportDB):
self._conn = self._open_export_db(self._dbfile)
self._insert_run_info()
def write_to_disk(self):
"""Write changes from in-memory database back to disk"""
# dump the database
conn = self._conn
conn.commit()
dbdump = self._dump_db(conn)
# cleanup the old on-disk database
# also unlink the wal and shm files if needed
dbfile = pathlib.Path(self._dbfile)
if dbfile.exists():
dbfile.unlink()
wal = dbfile.with_suffix(".db-wal")
if wal.exists():
wal.unlink()
shm = dbfile.with_suffix(".db-shm")
if shm.exists():
shm.unlink()
conn_on_disk = sqlite3.connect(str(dbfile))
conn_on_disk.cursor().executescript(dbdump.read())
conn_on_disk.commit()
conn_on_disk.close()
def close(self):
"""close the database connection"""
try:
if self._conn:
self._conn.close()
except Error as e:
logging.warning(e)
def _open_export_db(self, dbfile):
"""open export database and return a db connection
returns: connection to the database
@@ -588,21 +638,13 @@ class ExportDBInMemory(ExportDB):
self.was_created = True
self.was_upgraded = ()
else:
try:
conn = sqlite3.connect(dbfile)
except Error as e:
logging.warning(e)
raise e from e
tempfile = StringIO()
for line in conn.iterdump():
tempfile.write("%s\n" % line)
conn = sqlite3.connect(dbfile)
dbdump = self._dump_db(conn)
conn.close()
tempfile.seek(0)
# Create a database in memory and import from tempfile
# Create a database in memory and import from the dump
conn = sqlite3.connect(":memory:")
conn.cursor().executescript(tempfile.read())
conn.cursor().executescript(dbdump.read())
conn.commit()
self.was_created = False
version_info = self._get_database_version(conn)
@@ -625,6 +667,21 @@ class ExportDBInMemory(ExportDB):
return conn
def _dump_db(self, conn: sqlite3.Connection) -> StringIO:
"""dump sqlite db to a string buffer"""
dbdump = StringIO()
for line in conn.iterdump():
dbdump.write("%s\n" % line)
dbdump.seek(0)
return dbdump
def __del__(self):
"""close the database connection"""
try:
self.close()
except Error as e:
pass
class ExportDBTemp(ExportDBInMemory):
"""Temporary in-memory version of ExportDB"""

87
tests/test_cli.py
View File

@@ -6710,6 +6710,93 @@ def test_export_exportdb():
in result.output
)
def test_export_exportdb_ramdb():
"""test --exportdb --ramdb"""
import glob
import os
import os.path
import re
import osxphotos
from osxphotos.cli import export
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
result = runner.invoke(
export,
[os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--exportdb", "export.db", "--ramdb"],
)
assert result.exit_code == 0
assert re.search(r"Created export database.*export\.db", result.output)
files = glob.glob("*")
assert "export.db" in files
result = runner.invoke(
export,
[
os.path.join(cwd, CLI_PHOTOS_DB),
".",
"-V",
"--exportdb",
"export.db",
"--update",
"--ramdb"
],
)
assert result.exit_code == 0
assert re.search(r"Using export database.*export\.db", result.output)
assert "exported: 0" in result.output
def test_export_ramdb():
"""test --ramdb"""
import glob
import os
import os.path
import re
import osxphotos
from osxphotos.cli import export
runner = CliRunner()
cwd = os.getcwd()
# pylint: disable=not-context-manager
with runner.isolated_filesystem():
result = runner.invoke(
export,
[os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--ramdb"],
)
assert result.exit_code == 0
# run again, update should update no files if db written back to disk
result = runner.invoke(
export,
[
os.path.join(cwd, CLI_PHOTOS_DB),
".",
"-V",
"--update",
"--ramdb"
],
)
assert result.exit_code == 0
assert "exported: 0" in result.output
# run again without --ramdb, update should update no files if db written back to disk
result = runner.invoke(
export,
[
os.path.join(cwd, CLI_PHOTOS_DB),
".",
"-V",
"--update",
],
)
assert result.exit_code == 0
assert "exported: 0" in result.output
def test_export_finder_tag_keywords():
"""test --finder-tag-keywords"""

63
tests/test_export_db.py
View File

@@ -177,6 +177,69 @@ def test_export_db_in_memory():
assert uuids == [uuid]
def test_export_db_in_memory_write_to_disk():
"""test ExportDBInMemory with write back to disk"""
tempdir = tempfile.TemporaryDirectory(prefix="osxphotos_")
dbname = os.path.join(tempdir.name, ".osxphotos_export.db")
db = ExportDB(dbname, tempdir.name)
assert os.path.isfile(dbname)
filepath = os.path.join(tempdir.name, "test.JPG")
uuid = "FOOBAR"
record = db.create_file_record(filepath, uuid)
record.photoinfo = INFO_DATA
record.exifdata = EXIF_DATA
record.digest = DIGEST_DATA
record.src_sig = (7, 8, 9)
record.dest_sig = (10, 11, 12)
db.close()
# create in memory version
dbram = ExportDBInMemory(dbname, tempdir.name)
record2 = dbram.get_file_record(filepath)
assert record2.uuid == uuid
assert record2.photoinfo == INFO_DATA
assert record2.exifdata == EXIF_DATA
assert record2.digest == DIGEST_DATA
assert record2.src_sig == (7, 8, 9)
assert record2.dest_sig == (10, 11, 12)
# change some values
record2.photoinfo = INFO_DATA2
record2.exifdata = EXIF_DATA2
record2.digest = DIGEST_DATA2
record2.src_sig = (13, 14, 15)
record2.dest_sig = (16, 17, 18)
assert record2.photoinfo == INFO_DATA2
assert record2.exifdata == EXIF_DATA2
assert record2.digest == DIGEST_DATA2
assert record2.src_sig == (13, 14, 15)
assert record2.dest_sig == (16, 17, 18)
# all uuids
uuids = dbram.get_previous_uuids()
assert uuids == [uuid]
# write to disk
dbram.write_to_disk()
dbram.close()
# re-open original, assert changes are written back
db = ExportDB(dbname, tempdir.name)
record = db.get_file_record(filepath)
assert record.photoinfo == INFO_DATA2
assert record.exifdata == EXIF_DATA2
assert record.digest == DIGEST_DATA2
assert record.src_sig == (13, 14, 15)
assert record.dest_sig == (16, 17, 18)
# all uuids
uuids = db.get_previous_uuids()
assert uuids == [uuid]
def test_export_db_in_memory_nofile():
"""test ExportDBInMemory with no dbfile"""
tempdir = tempfile.TemporaryDirectory(prefix="osxphotos_")