From 76aee7f189b4b32e2e263a4e798711713ed17a14 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Tue, 4 Jan 2022 06:28:59 -0800 Subject: [PATCH] Export DB can now reside outside export directory, #568 --- README.md | 11 +++--- docs/.buildinfo | 2 +- docs/_static/documentation_options.js | 2 +- docs/cli.html | 2 +- docs/genindex.html | 2 +- docs/index.html | 2 +- docs/modules.html | 2 +- docs/reference.html | 2 +- docs/search.html | 2 +- osxphotos/_version.py | 2 +- osxphotos/cli.py | 29 +++++++-------- osxphotos/export_db.py | 52 ++++++++++++++------------- osxphotos/phototemplate.py | 4 ++- tests/test_cli.py | 12 +------ tests/test_export_db.py | 16 +++++---- 15 files changed, 67 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 74f55cda..c3ff8b55 100644 --- a/README.md +++ b/README.md @@ -1154,14 +1154,13 @@ Options: You can run more than one function by repeating the '--post-function' option with different arguments. See Post Function below. - --exportdb EXPORTDB_FILE Specify alternate name for database file which + --exportdb EXPORTDB_FILE Specify alternate path for database file which stores state information for export and --update. If --exportdb is not specified, export database will be saved to '.osxphotos_export.db' in the export - directory. Must be specified as filename - only, not a path, as export database will be - saved in export directory. + directory. If --exportdb is specified, it + will be saved to the specified file. --load-config Load options from file as written with --save- config. This allows you to save a complex @@ -1721,7 +1720,7 @@ Substitution Description {lf} A line feed: '\n', alias for {newline} {cr} A carriage return: '\r' {crlf} a carriage return + line feed: '\r\n' -{osxphotos_version} The osxphotos version, e.g. '0.44.3' +{osxphotos_version} The osxphotos version, e.g. '0.44.4' {osxphotos_cmd_line} The full command line used to run osxphotos The following substitutions may result in multiple values. Thus if specified for @@ -3623,7 +3622,7 @@ The following template field substitutions are availabe for use the templating s |{lf}|A line feed: '\n', alias for {newline}| |{cr}|A carriage return: '\r'| |{crlf}|a carriage return + line feed: '\r\n'| -|{osxphotos_version}|The osxphotos version, e.g. '0.44.3'| +|{osxphotos_version}|The osxphotos version, e.g. '0.44.4'| |{osxphotos_cmd_line}|The full command line used to run osxphotos| |{album}|Album(s) photo is contained in| |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder| diff --git a/docs/.buildinfo b/docs/.buildinfo index 012caedb..806a6bbf 100644 --- a/docs/.buildinfo +++ b/docs/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 8dc04ce2ac089dfa0c5fc3a14c14ed6e +config: abcd83bede460ffb3604a85d16e98db7 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_static/documentation_options.js b/docs/_static/documentation_options.js index 7dbf0e4d..5b1c1155 100644 --- a/docs/_static/documentation_options.js +++ b/docs/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '0.44.3', + VERSION: '0.44.4', LANGUAGE: 'None', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/docs/cli.html b/docs/cli.html index 696d25ce..6332fb3e 100644 --- a/docs/cli.html +++ b/docs/cli.html @@ -6,7 +6,7 @@ - osxphotos command line interface (CLI) — osxphotos 0.44.3 documentation + osxphotos command line interface (CLI) — osxphotos 0.44.4 documentation diff --git a/docs/genindex.html b/docs/genindex.html index f3c39980..be175daf 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -5,7 +5,7 @@ - Index — osxphotos 0.44.3 documentation + Index — osxphotos 0.44.4 documentation diff --git a/docs/index.html b/docs/index.html index 9ef05714..d8415bb8 100644 --- a/docs/index.html +++ b/docs/index.html @@ -6,7 +6,7 @@ - Welcome to osxphotos’s documentation! — osxphotos 0.44.3 documentation + Welcome to osxphotos’s documentation! — osxphotos 0.44.4 documentation diff --git a/docs/modules.html b/docs/modules.html index 6e4d2201..d7532a9e 100644 --- a/docs/modules.html +++ b/docs/modules.html @@ -6,7 +6,7 @@ - osxphotos — osxphotos 0.44.3 documentation + osxphotos — osxphotos 0.44.4 documentation diff --git a/docs/reference.html b/docs/reference.html index 7e5e2785..c95c2715 100644 --- a/docs/reference.html +++ b/docs/reference.html @@ -6,7 +6,7 @@ - osxphotos package — osxphotos 0.44.3 documentation + osxphotos package — osxphotos 0.44.4 documentation diff --git a/docs/search.html b/docs/search.html index e850277e..6f733658 100644 --- a/docs/search.html +++ b/docs/search.html @@ -5,7 +5,7 @@ - Search — osxphotos 0.44.3 documentation + Search — osxphotos 0.44.4 documentation diff --git a/osxphotos/_version.py b/osxphotos/_version.py index 8e643e7b..01c03a7a 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,3 +1,3 @@ """ version info """ -__version__ = "0.44.3" +__version__ = "0.44.4" diff --git a/osxphotos/cli.py b/osxphotos/cli.py index 217facaf..5d90376f 100644 --- a/osxphotos/cli.py +++ b/osxphotos/cli.py @@ -1060,10 +1060,9 @@ def cli(ctx, db, json_, debug): metavar="EXPORTDB_FILE", default=None, help=( - "Specify alternate name for database file which stores state information for export and --update. " + "Specify alternate path for database file which stores state information for export and --update. " f"If --exportdb is not specified, export database will be saved to '{OSXPHOTOS_EXPORT_DB}' " - "in the export directory. Must be specified as filename only, not a path, as export database " - "will be saved in export directory." + "in the export directory. If --exportdb is specified, it will be saved to the specified file. " ), type=click.Path(), ) @@ -1573,25 +1572,23 @@ def export( # sanity check exportdb if exportdb and exportdb != OSXPHOTOS_EXPORT_DB: - if "/" in exportdb: - click.echo( - click.style( - f"Error: --exportdb must be specified as filename not path; " - + f"export database will saved in export directory '{dest}'.", - fg=CLI_COLOR_ERROR, - ) - ) - raise click.Abort() - elif pathlib.Path(pathlib.Path(dest) / OSXPHOTOS_EXPORT_DB).exists(): + if pathlib.Path(pathlib.Path(dest) / OSXPHOTOS_EXPORT_DB).exists(): click.echo( click.style( f"Warning: export database is '{exportdb}' but found '{OSXPHOTOS_EXPORT_DB}' in {dest}; using '{exportdb}'", fg=CLI_COLOR_WARNING, ) ) + if pathlib.Path(exportdb).resolve().parent != pathlib.Path(dest): + click.echo( + click.style( + f"Warning: export database '{pathlib.Path(exportdb).resolve()}' is in a different directory than export destination '{dest}'", + fg=CLI_COLOR_WARNING, + ) + ) # open export database and assign copy/link/unlink functions - export_db_path = os.path.join(dest, exportdb or OSXPHOTOS_EXPORT_DB) + export_db_path = exportdb or os.path.join(dest, OSXPHOTOS_EXPORT_DB) # check that export isn't in the parent or child of a previously exported library other_db_files = find_files_in_branch(dest, OSXPHOTOS_EXPORT_DB) @@ -1614,10 +1611,10 @@ def export( click.confirm("Do you want to continue?", abort=True) if dry_run: - export_db = ExportDBInMemory(export_db_path) + export_db = ExportDBInMemory(dbfile=export_db_path, export_dir=dest) fileutil = FileUtilNoOp else: - export_db = ExportDB(export_db_path) + export_db = ExportDB(dbfile=export_db_path, export_dir=dest) fileutil = FileUtil if verbose_: diff --git a/osxphotos/export_db.py b/osxphotos/export_db.py index ec3e5d98..f3506680 100644 --- a/osxphotos/export_db.py +++ b/osxphotos/export_db.py @@ -1,5 +1,6 @@ """ Helper class for managing a database used by PhotoInfo.export for tracking state of exports and updates """ + import datetime import logging import os @@ -14,7 +15,7 @@ from ._constants import OSXPHOTOS_EXPORT_DB from ._version import __version__ OSXPHOTOS_EXPORTDB_VERSION = "4.0" -OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {str(datetime.datetime.now())}" +OSXPHOTOS_ABOUT_STRING = f"Created by osxphotos version {__version__} (https://github.com/RhetTbull/osxphotos) on {datetime.datetime.now()}" class ExportDB_ABC(ABC): @@ -171,7 +172,7 @@ class ExportDBNoOp(ExportDB_ABC): return [] def get_detected_text_for_uuid(self, uuid): - return None + return None def set_detected_text_for_uuid(self, uuid, json_text): pass @@ -193,15 +194,14 @@ class ExportDBNoOp(ExportDB_ABC): class ExportDB(ExportDB_ABC): """Interface to sqlite3 database used to store state information for osxphotos export command""" - def __init__(self, dbfile): + def __init__(self, dbfile, export_dir): """dbfile: path to osxphotos export database file""" self._dbfile = dbfile - # _path is parent of the database - # all files referenced by get_/set_uuid_for_file will be converted to - # relative paths to this parent _path + # export_dir is required as all files referenced by get_/set_uuid_for_file will be converted to + # relative paths to this path # this allows the entire export tree to be moved to a new disk/location # whilst preserving the UUID to filename mapping - self._path = pathlib.Path(dbfile).parent + self._path = export_dir self._conn = self._open_export_db(dbfile) self._insert_run_info() @@ -214,14 +214,13 @@ class ExportDB(ExportDB_ABC): try: c = conn.cursor() c.execute( - f"SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,) + "SELECT uuid FROM files WHERE filepath_normalized = ?", (filename,) ) results = c.fetchone() uuid = results[0] if results else None except Error as e: logging.warning(e) uuid = None - return uuid def set_uuid_for_file(self, filename, uuid): @@ -232,9 +231,10 @@ class ExportDB(ExportDB_ABC): try: c = conn.cursor() c.execute( - f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);", + "INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);", (filename, filename_normalized, uuid), ) + conn.commit() except Error as e: logging.warning(e) @@ -274,15 +274,14 @@ class ExportDB(ExportDB_ABC): ) results = c.fetchone() if results: - stats = results[0:3] + stats = results[:3] mtime = int(stats[2]) if stats[2] is not None else None stats = (stats[0], stats[1], mtime) else: stats = (None, None, None) except Error as e: logging.warning(e) - stats = (None, None, None) - + stats = None, None, None return stats def set_stat_edited_for_file(self, filename, stats): @@ -332,15 +331,14 @@ class ExportDB(ExportDB_ABC): ) results = c.fetchone() if results: - stats = results[0:3] + stats = results[:3] mtime = int(stats[2]) if stats[2] is not None else None stats = (stats[0], stats[1], mtime) else: stats = (None, None, None) except Error as e: logging.warning(e) - stats = (None, None, None) - + stats = None, None, None return stats def set_stat_converted_for_file(self, filename, stats): @@ -493,7 +491,10 @@ class ExportDB(ExportDB_ABC): c = conn.cursor() c.execute( "INSERT OR REPLACE INTO detected_text(uuid, text_data) VALUES (?, ?);", - (uuid, text_json,), + ( + uuid, + text_json, + ), ) conn.commit() except Error as e: @@ -517,9 +518,10 @@ class ExportDB(ExportDB_ABC): try: c = conn.cursor() c.execute( - f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);", + "INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);", (filename, filename_normalized, uuid), ) + c.execute( "UPDATE files " + "SET orig_mode = ?, orig_size = ?, orig_mtime = ? " @@ -582,7 +584,7 @@ class ExportDB(ExportDB_ABC): ) results = c.fetchone() if results: - stats = results[0:3] + stats = results[:3] mtime = int(stats[2]) if stats[2] is not None else None stats = (stats[0], stats[1], mtime) else: @@ -741,9 +743,10 @@ class ExportDB(ExportDB_ABC): try: c = conn.cursor() c.execute( - f"INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)", + "INSERT INTO runs (datetime, python_path, script_name, args, cwd) VALUES (?, ?, ?, ?, ?)", (dt, python_path, cmd, args, cwd), ) + conn.commit() except Error as e: logging.warning(e) @@ -755,14 +758,13 @@ class ExportDBInMemory(ExportDB): modifying the on-disk version """ - def __init__(self, dbfile): + def __init__(self, dbfile, export_dir): self._dbfile = dbfile or f"./{OSXPHOTOS_EXPORT_DB}" - # _path is parent of the database - # all files referenced by get_/set_uuid_for_file will be converted to - # relative paths to this parent _path + # export_dir is required as all files referenced by get_/set_uuid_for_file will be converted to + # relative paths to this path # this allows the entire export tree to be moved to a new disk/location # whilst preserving the UUID to filename mapping - self._path = pathlib.Path(self._dbfile).parent + self._path = export_dir self._conn = self._open_export_db(self._dbfile) self._insert_run_info() diff --git a/osxphotos/phototemplate.py b/osxphotos/phototemplate.py index e0e47396..8ec5f5ab 100644 --- a/osxphotos/phototemplate.py +++ b/osxphotos/phototemplate.py @@ -375,7 +375,9 @@ class PhotoTemplate: self.filepath = options.filepath self.quote = options.quote self.dest_path = options.dest_path - self.exportdb = options.exportdb or ExportDBInMemory(None) + self.exportdb = options.exportdb or ExportDBInMemory( + None, self.export_dir or "." + ) def render( self, diff --git a/tests/test_cli.py b/tests/test_cli.py index 24dbb85a..5f66fa35 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5610,7 +5610,7 @@ def test_export_ignore_signature_sidecar(): # change the sidecar data in export DB # should result in a new sidecar being exported but not the image itself - exportdb = osxphotos.export_db.ExportDB("./.osxphotos_export.db") + exportdb = osxphotos.export_db.ExportDB("./.osxphotos_export.db", ".") for filename in CLI_EXPORT_IGNORE_SIGNATURE_FILENAMES: exportdb.set_sidecar_for_file(f"{filename}.xmp", "FOO", (0, 1, 2)) @@ -6163,16 +6163,6 @@ def test_export_exportdb(): in result.output ) - # specify a path for exportdb, should generate error - result = runner.invoke( - export, - [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V", "--exportdb", "./export.db"], - ) - assert result.exit_code != 0 - assert ( - "Error: --exportdb must be specified as filename not path" in result.output - ) - def test_export_finder_tag_keywords(): """test --finder-tag-keywords""" diff --git a/tests/test_export_db.py b/tests/test_export_db.py index 98a82a19..db4445d4 100644 --- a/tests/test_export_db.py +++ b/tests/test_export_db.py @@ -22,7 +22,7 @@ def test_export_db(): tempdir = tempfile.TemporaryDirectory(prefix="osxphotos_") dbname = os.path.join(tempdir.name, ".osxphotos_export.db") - db = ExportDB(dbname) + db = ExportDB(dbname, tempdir.name) assert os.path.isfile(dbname) assert db.was_created assert not db.was_upgraded @@ -76,7 +76,7 @@ def test_export_db(): # close and re-open db.close() - db = ExportDB(dbname) + db = ExportDB(dbname, tempdir.name) assert not db.was_created assert db.get_uuid_for_file(filepath2) == "BAR-FOO" assert db.get_info_for_uuid("BAR-FOO") == INFO_DATA @@ -131,7 +131,7 @@ def test_export_db_no_op(): db.set_detected_text_for_uuid("FOO-BAR", json.dumps([["foo", 0.5]])) assert db.get_detected_text_for_uuid("FOO-BAR") is None - + # test set_data which sets all at the same time filepath2 = os.path.join(tempdir.name, "test2.jpg") db.set_data( @@ -171,7 +171,7 @@ def test_export_db_in_memory(): tempdir = tempfile.TemporaryDirectory(prefix="osxphotos_") dbname = os.path.join(tempdir.name, ".osxphotos_export.db") - db = ExportDB(dbname) + db = ExportDB(dbname, tempdir.name) assert os.path.isfile(dbname) filepath = os.path.join(tempdir.name, "test.JPG") @@ -190,7 +190,7 @@ def test_export_db_in_memory(): db.close() - dbram = ExportDBInMemory(dbname) + dbram = ExportDBInMemory(dbname, tempdir.name) assert not dbram.was_created assert not dbram.was_upgraded assert dbram.version == OSXPHOTOS_EXPORTDB_VERSION @@ -232,7 +232,7 @@ def test_export_db_in_memory(): dbram.close() # re-open on disk and verify no changes - db = ExportDB(dbname) + db = ExportDB(dbname, tempdir.name) assert db.get_uuid_for_file(filepath_lower) == "FOO-BAR" assert db.get_info_for_uuid("FOO-BAR") == INFO_DATA assert db.get_exifdata_for_file(filepath) == EXIF_DATA @@ -258,7 +258,9 @@ def test_export_db_in_memory_nofile(): filepath = os.path.join(tempdir.name, "test.JPG") filepath_lower = os.path.join(tempdir.name, "test.jpg") - dbram = ExportDBInMemory(os.path.join(tempdir.name, "NOT_A_DATABASE_FILE.db")) + dbram = ExportDBInMemory( + os.path.join(tempdir.name, "NOT_A_DATABASE_FILE.db"), tempdir.name + ) assert dbram.was_created assert not dbram.was_upgraded assert dbram.version == OSXPHOTOS_EXPORTDB_VERSION