From 5eb0876e331beb020431bb037dee75fb7ae61c85 Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 21:13:50 +0200 Subject: [PATCH 1/7] added --export-as-hardlink option --- README.md | 4 +++- osxphotos/__main__.py | 14 +++++++++++++- osxphotos/photoinfo.py | 18 +++++++++++++++--- osxphotos/utils.py | 26 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4d3001c1..27630cbe 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ Options: Search by end item date, e.g. 2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 w/o TZ). + --export-as-hardlink Hardlink files instead of copying them. --overwrite Overwrite existing files. Default behavior is to add (1), (2), etc to filename if file already exists. Use this with caution as it @@ -984,12 +985,13 @@ Returns True if photo is a panorama, otherwise False. Returns a JSON representation of all photo info #### `export()` -`export(dest, *filename, edited=False, live_photo=False, overwrite=False, increment=True, sidecar_json=False, sidecar_xmp=False, use_photos_export=False, timeout=120, exiftool=False, no_xattr=False, use_albums_as_keywords=False, use_persons_as_keywords=False)` +`export(dest, *filename, edited=False, live_photo=False, export_as_hardlink=False, overwrite=False, increment=True, sidecar_json=False, sidecar_xmp=False, use_photos_export=False, timeout=120, exiftool=False, no_xattr=False, use_albums_as_keywords=False, use_persons_as_keywords=False)` Export photo from the Photos library to another destination on disk. - dest: must be valid destination path as str (or exception raised). - *filename (optional): name of picture as str; if not provided, will use current filename. **NOTE**: if provided, user must ensure file extension (suffix) is correct. For example, if photo is .CR2 file, edited image may be .jpeg. If you provide an extension different than what the actual file is, export will print a warning but will happily export the photo using the incorrect file extension. e.g. to get the extension of the edited photo, look at [PhotoInfo.path_edited](#path_edited). - edited: boolean; if True (default=False), will export the edited version of the photo (or raise exception if no edited version) +- export_as_hardlink: boolean; if True (default=False), will hardlink files instead of copying them - overwrite: boolean; if True (default=False), will overwrite files if they alreay exist - live_photo: boolean; if True (default=False), will also export the associted .mov for live photos; exported live photo will be named filename.mov - increment: boolean; if True (default=True), will increment file name until a non-existent name is found diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index d8c0cdae..a16c94dc 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -89,7 +89,7 @@ class ExportCommand(click.Command): ) formatter.write("\n") formatter.write_text( - "The templating system may also be used with the --keyword-template option " + "The templating system may also be used with the --keyword-template option " + "to set keywords on export (with --exiftool or --sidecar), " + "for example, to set a new keyword in format 'folder/subfolder/album' to " + 'preserve the folder/album structure, you can use --keyword-template "{folder_album}"' @@ -860,6 +860,11 @@ def query( @DB_OPTION @click.option("--verbose", "-V", is_flag=True, help="Print verbose output.") @query_options +@click.option( + "--export-as-hardlink", + is_flag=True, + help="Hardlink files instead of copying them. ", +) @click.option( "--overwrite", is_flag=True, @@ -1004,6 +1009,7 @@ def export( from_date, to_date, verbose, + export_as_hardlink, overwrite, export_by_date, skip_edited, @@ -1195,6 +1201,7 @@ def export( verbose, export_by_date, sidecar, + export_as_hardlink, overwrite, export_edited, original_name, @@ -1216,6 +1223,7 @@ def export( verbose, export_by_date, sidecar, + export_as_hardlink, overwrite, export_edited, original_name, @@ -1605,6 +1613,7 @@ def export_photo( verbose, export_by_date, sidecar, + export_as_hardlink, overwrite, export_edited, original_name, @@ -1624,6 +1633,7 @@ def export_photo( verbose: boolean; print verbose output export_by_date: boolean; create export folder in form dest/YYYY/MM/DD sidecar: list zero, 1 or 2 of ["json","xmp"] of sidecar variety to export + export_as_hardlink: boolean; hardlink files instead of copying them overwrite: boolean; overwrite dest file if it already exists original_name: boolean; use original filename instead of current filename export_live: boolean; also export live video component if photo is a live photo @@ -1715,6 +1725,7 @@ def export_photo( sidecar_xmp=sidecar_xmp, live_photo=export_live, raw_photo=export_raw, + export_as_hardlink=export_as_hardlink, overwrite=overwrite, use_photos_export=use_photos_export, exiftool=exiftool, @@ -1752,6 +1763,7 @@ def export_photo( edited_name, sidecar_json=sidecar_json, sidecar_xmp=sidecar_xmp, + export_as_hardlink=export_as_hardlink, overwrite=overwrite, edited=True, use_photos_export=use_photos_export, diff --git a/osxphotos/photoinfo.py b/osxphotos/photoinfo.py index 1df32457..dac2874f 100644 --- a/osxphotos/photoinfo.py +++ b/osxphotos/photoinfo.py @@ -40,6 +40,7 @@ from .template import ( TEMPLATE_SUBSTITUTIONS_MULTI_VALUED, ) from .utils import ( + _hardlink_file, _copy_file, _export_photo_uuid_applescript, _get_resource_loc, @@ -635,6 +636,7 @@ class PhotoInfo: edited=False, live_photo=False, raw_photo=False, + export_as_hardlink=False, overwrite=False, increment=True, sidecar_json=False, @@ -660,6 +662,7 @@ class PhotoInfo: (or raise exception if no edited version) live_photo: (boolean, default=False); if True, will also export the associted .mov for live photos raw_photo: (boolean, default=False); if True, will also export the associted RAW photo + export_as_hardlink: (boolean, default=False); if True, will hardlink files instead of copying them overwrite: (boolean, default=False); if True will overwrite files if they alreay exist increment: (boolean, default=True); if True, will increment file name until a non-existant name is found if overwrite=False and increment=False, export will fail if destination file already exists @@ -802,7 +805,10 @@ class PhotoInfo: ) # copy the file, _copy_file uses ditto to preserve Mac extended attributes - _copy_file(src, dest, norsrc=no_xattr) + if export_as_hardlink: + _hardlink_file(src, dest) + else: + _copy_file(src, dest, norsrc=no_xattr) exported_files.append(str(dest)) # copy live photo associated .mov if requested @@ -814,7 +820,10 @@ class PhotoInfo: logging.debug( f"Exporting live photo video of {filename} as {live_name.name}" ) - _copy_file(src_live, str(live_name), norsrc=no_xattr) + if export_as_hardlink: + _hardlink_file(src_live, str(live_name)) + else: + _copy_file(src_live, str(live_name), norsrc=no_xattr) exported_files.append(str(live_name)) else: logging.warning(f"Skipping missing live movie for {filename}") @@ -828,7 +837,10 @@ class PhotoInfo: logging.debug( f"Exporting RAW photo of {filename} as {raw_name.name}" ) - _copy_file(str(raw_path), str(raw_name), norsrc=no_xattr) + if export_as_hardlink: + _hardlink_file(str(raw_path), str(raw_name)) + else: + _copy_file(str(raw_path), str(raw_name), norsrc=no_xattr) exported_files.append(str(raw_name)) else: logging.warning(f"Skipping missing RAW photo for {filename}") diff --git a/osxphotos/utils.py b/osxphotos/utils.py index ef1aede5..c87d1810 100644 --- a/osxphotos/utils.py +++ b/osxphotos/utils.py @@ -118,6 +118,32 @@ def _dd_to_dms(dd): return int(deg_), int(min_), sec_ +def _hardlink_file(src, dest): + """ Hardlinks a file from src path to dest path + src: source path as string + dest: destination path as string + Raises exception if linking fails or either path is None """ + + if src is None or dest is None: + raise ValueError("src and dest must not be None", src, dest) + + if not os.path.isfile(src): + raise FileNotFoundError("src file does not appear to exist", src) + + command = ["ln", src, dest] + + # if error on copy, subprocess will raise CalledProcessError + try: + result = subprocess.run(command, check=True, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + logging.critical( + f"ln returned error: {e.returncode} {e.stderr.decode(sys.getfilesystemencoding()).rstrip()}" + ) + raise e + + return result.returncode + + def _copy_file(src, dest, norsrc=False): """ Copies a file from src path to dest path src: source path as string From 69356c0b57efa04cb4fc92fd8dd0a8821a7f295c Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 22:55:56 +0200 Subject: [PATCH 2/7] Update osxphotos/utils.py Co-authored-by: Rhet Turnbull --- osxphotos/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/osxphotos/utils.py b/osxphotos/utils.py index c87d1810..2dffa579 100644 --- a/osxphotos/utils.py +++ b/osxphotos/utils.py @@ -130,7 +130,6 @@ def _hardlink_file(src, dest): if not os.path.isfile(src): raise FileNotFoundError("src file does not appear to exist", src) - command = ["ln", src, dest] # if error on copy, subprocess will raise CalledProcessError try: From cceab62993930a90a0236b7b36ddf4553bd0e867 Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 22:56:12 +0200 Subject: [PATCH 3/7] Update osxphotos/utils.py Co-authored-by: Rhet Turnbull --- osxphotos/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osxphotos/utils.py b/osxphotos/utils.py index 2dffa579..d54090d3 100644 --- a/osxphotos/utils.py +++ b/osxphotos/utils.py @@ -133,7 +133,7 @@ def _hardlink_file(src, dest): # if error on copy, subprocess will raise CalledProcessError try: - result = subprocess.run(command, check=True, stderr=subprocess.PIPE) + os.link(src, dest) except subprocess.CalledProcessError as e: logging.critical( f"ln returned error: {e.returncode} {e.stderr.decode(sys.getfilesystemencoding()).rstrip()}" From a8622b6b908aef4a02e9a16d18d22f3afcd43c13 Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 22:56:24 +0200 Subject: [PATCH 4/7] Update osxphotos/utils.py Co-authored-by: Rhet Turnbull --- osxphotos/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osxphotos/utils.py b/osxphotos/utils.py index d54090d3..d478e188 100644 --- a/osxphotos/utils.py +++ b/osxphotos/utils.py @@ -134,7 +134,7 @@ def _hardlink_file(src, dest): # if error on copy, subprocess will raise CalledProcessError try: os.link(src, dest) - except subprocess.CalledProcessError as e: + except Exception as e: logging.critical( f"ln returned error: {e.returncode} {e.stderr.decode(sys.getfilesystemencoding()).rstrip()}" ) From 97d3c69adee220001d378e72c96fc7ab6d91f44f Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 22:56:43 +0200 Subject: [PATCH 5/7] Update osxphotos/utils.py Co-authored-by: Rhet Turnbull --- osxphotos/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/osxphotos/utils.py b/osxphotos/utils.py index d478e188..37c30c43 100644 --- a/osxphotos/utils.py +++ b/osxphotos/utils.py @@ -140,7 +140,6 @@ def _hardlink_file(src, dest): ) raise e - return result.returncode def _copy_file(src, dest, norsrc=False): From cb124713d63c6999ccf97cc03ab241d8a569c470 Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 23:06:32 +0200 Subject: [PATCH 6/7] version increment after adding --export-as-hardlink option --- osxphotos/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osxphotos/_version.py b/osxphotos/_version.py index 9815fa64..2a780cb0 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,3 +1,3 @@ """ version info """ -__version__ = "0.28.13" +__version__ = "0.28.14" From b0ec6c6b36d8cfe05723d47b210d9d7c5aabdfe5 Mon Sep 17 00:00:00 2001 From: britiscurious Date: Fri, 8 May 2020 23:29:18 +0200 Subject: [PATCH 7/7] added test for export using hardlinks, fixed a test that failed if users locale settings were different to en_US --- tests/test_cli.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index b106855a..da3b592a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -241,6 +241,23 @@ def test_export(): assert sorted(files) == sorted(CLI_EXPORT_FILENAMES) +def test_export_using_hardlinks(): + import glob + import os + import os.path + import osxphotos + from osxphotos.__main__ import export + + runner = CliRunner() + cwd = os.getcwd() + # pylint: disable=not-context-manager + with runner.isolated_filesystem(): + result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "--export-as-hardlink","-V"]) + assert result.exit_code == 0 + files = glob.glob("*") + assert sorted(files) == sorted(CLI_EXPORT_FILENAMES) + + def test_export_current_name(): import glob import os @@ -473,11 +490,14 @@ def test_export_raw_edited_original(): def test_export_directory_template_1(): # test export using directory template import glob + import locale import os import os.path import osxphotos from osxphotos.__main__ import export + locale.setlocale(locale.LC_ALL, "en_US") + runner = CliRunner() cwd = os.getcwd() # pylint: disable=not-context-manager