forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Performance improvements, added --profile

Browse Source
This commit is contained in:
Rhet Turnbull
2022-01-23 17:14:55 -08:00
parent 61a300250d
commit 74868238f3
14 changed files with 173 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -1723,7 +1723,7 @@ Substitution Description
{lf} A line feed: '\n', alias for {newline} {lf} A line feed: '\n', alias for {newline}
{cr} A carriage return: '\r' {cr} A carriage return: '\r'
{crlf} a carriage return + line feed: '\r\n' {crlf} a carriage return + line feed: '\r\n'
{osxphotos_version} The osxphotos version, e.g. '0.44.10' {osxphotos_version} The osxphotos version, e.g. '0.44.11'
{osxphotos_cmd_line} The full command line used to run osxphotos {osxphotos_cmd_line} The full command line used to run osxphotos
The following substitutions may result in multiple values. Thus if specified for The following substitutions may result in multiple values. Thus if specified for
@@ -3627,7 +3627,7 @@ The following template field substitutions are availabe for use the templating s
|{lf}|A line feed: '\n', alias for {newline}| |{lf}|A line feed: '\n', alias for {newline}|
|{cr}|A carriage return: '\r'| |{cr}|A carriage return: '\r'|
|{crlf}|a carriage return + line feed: '\r\n'| |{crlf}|a carriage return + line feed: '\r\n'|
|{osxphotos_version}|The osxphotos version, e.g. '0.44.10'| |{osxphotos_version}|The osxphotos version, e.g. '0.44.11'|
|{osxphotos_cmd_line}|The full command line used to run osxphotos| |{osxphotos_cmd_line}|The full command line used to run osxphotos|
|{album}|Album(s) photo is contained in| |{album}|Album(s) photo is contained in|
|{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder| |{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|

2
docs/.buildinfo
View File

@@ -1,4 +1,4 @@
# Sphinx build info version 1 # Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 716e0bf3a38e2e923691f2db50ed7ba7 config: dbae0cb30116fdf5326586184a4e5e1e
tags: 645f666f9bcd5a90fca523b33c5a78b7 tags: 645f666f9bcd5a90fca523b33c5a78b7

2
docs/_static/documentation_options.js vendored
View File

@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = { var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '0.44.10', VERSION: '0.44.11',
LANGUAGE: 'None', LANGUAGE: 'None',
COLLAPSE_INDEX: false, COLLAPSE_INDEX: false,
BUILDER: 'html', BUILDER: 'html',

2
docs/cli.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.10 documentation</title> <title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script> <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/genindex.html
View File

@@ -5,7 +5,7 @@
<head> <head>
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Index &#8212; osxphotos 0.44.10 documentation</title> <title>Index &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script> <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/index.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.10 documentation</title> <title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script> <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/modules.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos &#8212; osxphotos 0.44.10 documentation</title> <title>osxphotos &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script> <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/reference.html
View File

@@ -6,7 +6,7 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos package &#8212; osxphotos 0.44.10 documentation</title> <title>osxphotos package &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script> <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/search.html
View File

@@ -5,7 +5,7 @@
<head> <head>
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Search &#8212; osxphotos 0.44.10 documentation</title> <title>Search &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" /> <link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" /> <link rel="stylesheet" type="text/css" href="_static/alabaster.css" />

38
osxphotos/__init__.py
View File

@@ -1,13 +1,45 @@
from ._constants import AlbumSortOrder from ._constants import AlbumSortOrder
from ._version import __version__ from ._version import __version__
from .exiftool import ExifTool from .exiftool import ExifTool
from .photoexporter import ExportResults, PhotoExporter from .export_db import ExportDB, ExportDBInMemory, ExportDBNoOp
from .fileutil import FileUtil, FileUtilNoOp
from .momentinfo import MomentInfo
from .personinfo import PersonInfo
from .photoexporter import ExportOptions, ExportResults, PhotoExporter
from .photoinfo import PhotoInfo from .photoinfo import PhotoInfo
from .photosdb import PhotosDB from .photosdb import PhotosDB
from .photosdb._photosdb_process_comments import CommentInfo, LikeInfo from .photosdb._photosdb_process_comments import CommentInfo, LikeInfo
from .phototemplate import PhotoTemplate from .phototemplate import PhotoTemplate
from .placeinfo import PlaceInfo
from .queryoptions import QueryOptions from .queryoptions import QueryOptions
from .scoreinfo import ScoreInfo
from .searchinfo import SearchInfo
from .utils import _debug, _get_logger, _set_debug from .utils import _debug, _get_logger, _set_debug
# TODO: Add test for imageTimeZoneOffsetSeconds = None __all__ = [
# TODO: Add special albums and magic albums "__version__",
"_debug",
"_get_logger",
"_set_debug",
"AlbumSortOrder",
"CommentInfo",
"ExifTool",
"ExportDB",
"ExportDBInMemory",
"ExportDBNoOp",
"ExportOptions",
"ExportResults",
"FileUtil",
"FileUtilNoOp",
"LikeInfo",
"MomentInfo",
"PersonInfo",
"PhotoExporter",
"PhotoInfo",
"PhotosDB",
"PhotoTemplate",
"PlaceInfo",
"QueryOptions",
"ScoreInfo",
"SearchInfo",
]

18
osxphotos/_constants.py
View File

@@ -305,3 +305,21 @@ class AlbumSortOrder(Enum):
TEXT_DETECTION_CONFIDENCE_THRESHOLD = 0.75 TEXT_DETECTION_CONFIDENCE_THRESHOLD = 0.75
# stat sort order for cProfile: https://docs.python.org/3/library/profile.html#pstats.Stats.sort_stats
PROFILE_SORT_KEYS = [
"calls",
"cumulative",
"cumtime",
"file",
"filename",
"module",
"ncalls",
"pcalls",
"line",
"name",
"nfl",
"stdname",
"time",
"tottime",
]

2
osxphotos/_version.py
View File

@@ -1,3 +1,3 @@
""" version info """ """ version info """
__version__ = "0.44.10" __version__ = "0.44.11"

86
osxphotos/cli.py
View File

@@ -1,19 +1,24 @@
"""Command line interface for osxphotos """ """Command line interface for osxphotos """
import atexit
import code import code
import cProfile
import csv import csv
import datetime import datetime
import io
import json import json
import os import os
import os.path import os.path
import pathlib import pathlib
import pprint import pprint
import pstats
import shlex import shlex
import shutil import shutil
import subprocess import subprocess
import sys import sys
import time import time
from runpy import run_module from runpy import run_module
from typing import Dict
import bitmath import bitmath
import click import click
@@ -43,6 +48,7 @@ from ._constants import (
OSXPHOTOS_EXPORT_DB, OSXPHOTOS_EXPORT_DB,
OSXPHOTOS_URL, OSXPHOTOS_URL,
POST_COMMAND_CATEGORIES, POST_COMMAND_CATEGORIES,
PROFILE_SORT_KEYS,
SIDECAR_EXIFTOOL, SIDECAR_EXIFTOOL,
SIDECAR_JSON, SIDECAR_JSON,
SIDECAR_XMP, SIDECAR_XMP,
@@ -69,7 +75,12 @@ from .pyrepl import embed_repl
from .queryoptions import QueryOptions from .queryoptions import QueryOptions
from .sqlgrep import sqlgrep from .sqlgrep import sqlgrep
from .uti import get_preferred_uti_extension from .uti import get_preferred_uti_extension
from .utils import expand_and_validate_filepath, load_function, normalize_fs_path from .utils import (
expand_and_validate_filepath,
list_directory,
load_function,
normalize_fs_path,
)
__all__ = [ __all__ = [
"verbose_", "verbose_",
@@ -121,6 +132,9 @@ __all__ = [
# set via --verbose/-V # set via --verbose/-V
VERBOSE = False VERBOSE = False
# used to show/hide hidden commands
OSXPHOTOS_HIDDEN = not bool(os.getenv("OSXPHOTOS_SHOW_HIDDEN", default=False))
# used by snap and diff commands # used by snap and diff commands
OSXPHOTOS_SNAPSHOT_DIR = "/private/tmp/osxphotos_snapshots" OSXPHOTOS_SNAPSHOT_DIR = "/private/tmp/osxphotos_snapshots"
@@ -645,7 +659,9 @@ def QUERY_OPTIONS(f):
@click.group(context_settings=CTX_SETTINGS) @click.group(context_settings=CTX_SETTINGS)
@DB_OPTION @DB_OPTION
@JSON_OPTION @JSON_OPTION
@click.option("--debug", required=False, is_flag=True, default=False, hidden=True) @click.option(
"--debug", required=False, is_flag=True, default=False, hidden=OSXPHOTOS_HIDDEN
)
@click.version_option(__version__, "--version", "-v") @click.version_option(__version__, "--version", "-v")
@click.pass_context @click.pass_context
def cli(ctx, db, json_, debug): def cli(ctx, db, json_, debug):
@@ -1144,7 +1160,32 @@ def cli(ctx, db, json_, debug):
type=click.Path(), type=click.Path(),
) )
@click.option( @click.option(
"--beta", is_flag=True, default=False, hidden=True, help="Enable beta options." "--beta",
is_flag=True,
default=False,
hidden=OSXPHOTOS_HIDDEN,
help="Enable beta options.",
)
@click.option(
"--profile",
is_flag=True,
default=False,
hidden=OSXPHOTOS_HIDDEN,
help="Run export with code profiler.",
)
@click.option(
"--profile-sort",
default=None,
hidden=OSXPHOTOS_HIDDEN,
multiple=True,
metavar="SORT_KEY",
type=click.Choice(
PROFILE_SORT_KEYS,
case_sensitive=True,
),
help="Sort profiler output by SORT_KEY as specified at https://docs.python.org/3/library/profile.html#pstats.Stats.sort_stats. "
f"Can be specified multiple times. Valid options are: {PROFILE_SORT_KEYS}. "
"Default = 'cumulative'.",
) )
@DB_ARGUMENT @DB_ARGUMENT
@click.argument("dest", nargs=1, type=click.Path(exists=True)) @click.argument("dest", nargs=1, type=click.Path(exists=True))
@@ -1284,6 +1325,8 @@ def export(
preview, preview,
preview_suffix, preview_suffix,
preview_if_missing, preview_if_missing,
profile,
profile_sort,
): ):
"""Export photos from the Photos database. """Export photos from the Photos database.
Export path DEST is required. Export path DEST is required.
@@ -1297,6 +1340,24 @@ def export(
to modify this behavior. to modify this behavior.
""" """
if profile:
click.echo("Profiling...")
profile_sort = profile_sort or ["cumulative"]
click.echo(f"Profile sort_stats order: {profile_sort}")
pr = cProfile.Profile()
pr.enable()
def at_exit():
pr.disable()
click.echo("Profiling completed")
s = io.StringIO()
pstats.Stats(pr, stream=s).strip_dirs().sort_stats(
*profile_sort
).print_stats()
click.echo(s.getvalue())
atexit.register(at_exit)
# NOTE: because of the way ConfigOptions works, Click options must not # NOTE: because of the way ConfigOptions works, Click options must not
# set defaults which are not None or False. If defaults need to be set # set defaults which are not None or False. If defaults need to be set
# do so below after load_config and save_config are handled. # do so below after load_config and save_config are handled.
@@ -2046,6 +2107,18 @@ def export(
export_db.close() export_db.close()
def _export_with_profiler(args: Dict):
""" "Run export with cProfile"""
try:
args.pop("profile")
except KeyError:
pass
cProfile.runctx(
"_export(**args)", globals=globals(), locals=locals(), sort="tottime"
)
@cli.command() @cli.command()
@click.argument("topic", default=None, required=False, nargs=1) @click.argument("topic", default=None, required=False, nargs=1)
@click.pass_context @click.pass_context
@@ -3457,8 +3530,7 @@ def cleanup_files(dest_path, files_to_keep, fileutil):
deleted_files = [] deleted_files = []
for p in pathlib.Path(dest_path).rglob("*"): for p in pathlib.Path(dest_path).rglob("*"):
path = normalize_fs_path(str(p).lower()) if p.is_file() and normalize_fs_path(str(p).lower()) not in keepers:
if p.is_file() and path not in keepers:
verbose_(f"Deleting {p}") verbose_(f"Deleting {p}")
fileutil.unlink(p) fileutil.unlink(p)
deleted_files.append(str(p)) deleted_files.append(str(p))
@@ -4272,7 +4344,7 @@ def repl(ctx, cli_obj, db, emacs):
) )
@cli.command(name="grep", hidden=True) @cli.command(name="grep", hidden=OSXPHOTOS_HIDDEN)
@DB_OPTION @DB_OPTION
@click.pass_obj @click.pass_obj
@click.pass_context @click.pass_context
@@ -4318,7 +4390,7 @@ def grep(ctx, cli_obj, db, ignore_case, print_filename, pattern):
print(", ".join([table, column, row_id, value])) print(", ".join([table, column, row_id, value]))
@cli.command(hidden=True) @cli.command(hidden=OSXPHOTOS_HIDDEN)
@DB_OPTION @DB_OPTION
@DB_ARGUMENT @DB_ARGUMENT
@click.option( @click.option(

48
osxphotos/utils.py
View File

@@ -16,28 +16,29 @@ import sys
import unicodedata import unicodedata
import urllib.parse import urllib.parse
from plistlib import load as plistload from plistlib import load as plistload
from typing import Callable, Union from typing import Callable, List, Union
import CoreFoundation import CoreFoundation
import objc import objc
from Foundation import NSString from Foundation import NSFileManager, NSString
from ._constants import UNICODE_FORMAT from ._constants import UNICODE_FORMAT
__all__ = [ __all__ = [
"noop",
"lineno",
"dd_to_dms_str", "dd_to_dms_str",
"get_system_library_path", "expand_and_validate_filepath",
"get_last_library_path",
"list_photo_libraries",
"normalize_fs_path",
"findfiles", "findfiles",
"normalize_unicode", "get_last_library_path",
"get_system_library_path",
"increment_filename_with_count", "increment_filename_with_count",
"increment_filename", "increment_filename",
"expand_and_validate_filepath", "lineno",
"list_directory",
"list_photo_libraries",
"load_function", "load_function",
"noop",
"normalize_fs_path",
"normalize_unicode",
] ]
_DEBUG = False _DEBUG = False
@@ -299,10 +300,21 @@ def findfiles(pattern, path_):
# paths need to be normalized for unicode as filesystem returns unicode in NFD form # paths need to be normalized for unicode as filesystem returns unicode in NFD form
pattern = normalize_fs_path(pattern) pattern = normalize_fs_path(pattern)
rule = re.compile(fnmatch.translate(pattern), re.IGNORECASE) rule = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
files = [normalize_fs_path(p) for p in os.listdir(path_)] files = list_directory(path_)
return [name for name in files if rule.match(name)] return [name for name in files if rule.match(name)]
def list_directory(directory_path: str) -> List[str]:
"""List directory contents using NSFileManager"""
"""[[NSFileManager defaultManager] contentsOfDirectoryAtPath:@"directoryName" error:nil]"""
with objc.autorelease_pool():
manager = NSFileManager.defaultManager()
contents, error = manager.contentsOfDirectoryAtPath_error_(directory_path, None)
if error:
raise OSError(f"Error listing directory {directory_path}: {error}")
return [str(path) for path in contents]
def _open_sql_file(dbname): def _open_sql_file(dbname):
"""opens sqlite file dbname in read-only mode """opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor)""" returns tuple of (connection, cursor)"""
@@ -400,15 +412,15 @@ def increment_filename_with_count(
""" """
dest = filepath if isinstance(filepath, pathlib.Path) else pathlib.Path(filepath) dest = filepath if isinstance(filepath, pathlib.Path) else pathlib.Path(filepath)
dest_files = findfiles(f"{dest.stem}*", str(dest.parent)) dest_files = findfiles(f"{dest.stem}*", str(dest.parent))
dest_files = [normalize_fs_path(pathlib.Path(f).stem.lower()) for f in dest_files] dest_files = [pathlib.Path(f).stem.lower() for f in dest_files]
dest_new = dest.stem dest_new = f"{dest.stem} ({count})" if count else dest.stem
if count: dest_new = normalize_fs_path(dest_new)
dest_new = f"{dest.stem} ({count})"
while normalize_fs_path(dest_new.lower()) in dest_files: while dest_new.lower() in dest_files:
count += 1 count += 1
dest_new = f"{dest.stem} ({count})" dest_new = normalize_fs_path(f"{dest.stem} ({count})")
dest = dest.parent / f"{dest_new}{dest.suffix}" dest = dest.parent / f"{dest_new}{dest.suffix}"
return str(dest), count return normalize_fs_path(str(dest)), count
def increment_filename(filepath: Union[str, pathlib.Path]) -> str: def increment_filename(filepath: Union[str, pathlib.Path]) -> str: