forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Performance improvements, added --profile

Browse Source
This commit is contained in:
Rhet Turnbull 2022-01-23 17:14:55 -08:00
parent 61a300250d
commit 74868238f3
14 changed files with 173 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -1723,7 +1723,7 @@ Substitution Description
{lf} A line feed: '\n', alias for {newline}
{cr} A carriage return: '\r'
{crlf} a carriage return + line feed: '\r\n'
{osxphotos_version} The osxphotos version, e.g. '0.44.10'
{osxphotos_version} The osxphotos version, e.g. '0.44.11'
{osxphotos_cmd_line} The full command line used to run osxphotos
The following substitutions may result in multiple values. Thus if specified for
@ -3627,7 +3627,7 @@ The following template field substitutions are availabe for use the templating s
|{lf}|A line feed: '\n', alias for {newline}|
|{cr}|A carriage return: '\r'|
|{crlf}|a carriage return + line feed: '\r\n'|
|{osxphotos_version}|The osxphotos version, e.g. '0.44.10'|
|{osxphotos_version}|The osxphotos version, e.g. '0.44.11'|
|{osxphotos_cmd_line}|The full command line used to run osxphotos|
|{album}|Album(s) photo is contained in|
|{folder_album}|Folder path + album photo is contained in. e.g. 'Folder/Subfolder/Album' or just 'Album' if no enclosing folder|

2
docs/.buildinfo
View File

@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 716e0bf3a38e2e923691f2db50ed7ba7
config: dbae0cb30116fdf5326586184a4e5e1e
tags: 645f666f9bcd5a90fca523b33c5a78b7

2
docs/_static/documentation_options.js vendored
View File

@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '0.44.10',
VERSION: '0.44.11',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
BUILDER: 'html',

2
docs/cli.html
View File

@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.10 documentation</title>
<title>osxphotos command line interface (CLI) &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/genindex.html
View File

@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Index &#8212; osxphotos 0.44.10 documentation</title>
<title>Index &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/index.html
View File

@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.10 documentation</title>
<title>Welcome to osxphotos’s documentation! &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/modules.html
View File

@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos &#8212; osxphotos 0.44.10 documentation</title>
<title>osxphotos &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/reference.html
View File

@ -6,7 +6,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>osxphotos package &#8212; osxphotos 0.44.10 documentation</title>
<title>osxphotos package &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>

2
docs/search.html
View File

@ -5,7 +5,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Search &#8212; osxphotos 0.44.10 documentation</title>
<title>Search &#8212; osxphotos 0.44.11 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/alabaster.css" />

38
osxphotos/__init__.py
View File

@ -1,13 +1,45 @@
from ._constants import AlbumSortOrder
from ._version import __version__
from .exiftool import ExifTool
from .photoexporter import ExportResults, PhotoExporter
from .export_db import ExportDB, ExportDBInMemory, ExportDBNoOp
from .fileutil import FileUtil, FileUtilNoOp
from .momentinfo import MomentInfo
from .personinfo import PersonInfo
from .photoexporter import ExportOptions, ExportResults, PhotoExporter
from .photoinfo import PhotoInfo
from .photosdb import PhotosDB
from .photosdb._photosdb_process_comments import CommentInfo, LikeInfo
from .phototemplate import PhotoTemplate
from .placeinfo import PlaceInfo
from .queryoptions import QueryOptions
from .scoreinfo import ScoreInfo
from .searchinfo import SearchInfo
from .utils import _debug, _get_logger, _set_debug
# TODO: Add test for imageTimeZoneOffsetSeconds = None
# TODO: Add special albums and magic albums
__all__ = [
"__version__",
"_debug",
"_get_logger",
"_set_debug",
"AlbumSortOrder",
"CommentInfo",
"ExifTool",
"ExportDB",
"ExportDBInMemory",
"ExportDBNoOp",
"ExportOptions",
"ExportResults",
"FileUtil",
"FileUtilNoOp",
"LikeInfo",
"MomentInfo",
"PersonInfo",
"PhotoExporter",
"PhotoInfo",
"PhotosDB",
"PhotoTemplate",
"PlaceInfo",
"QueryOptions",
"ScoreInfo",
"SearchInfo",
]

18
osxphotos/_constants.py
View File

@ -305,3 +305,21 @@ class AlbumSortOrder(Enum):
TEXT_DETECTION_CONFIDENCE_THRESHOLD = 0.75
# stat sort order for cProfile: https://docs.python.org/3/library/profile.html#pstats.Stats.sort_stats
PROFILE_SORT_KEYS = [
"calls",
"cumulative",
"cumtime",
"file",
"filename",
"module",
"ncalls",
"pcalls",
"line",
"name",
"nfl",
"stdname",
"time",
"tottime",
]

2
osxphotos/_version.py
View File

@ -1,3 +1,3 @@
""" version info """
__version__ = "0.44.10"
__version__ = "0.44.11"

86
osxphotos/cli.py
View File

@ -1,19 +1,24 @@
"""Command line interface for osxphotos """
import atexit
import code
import cProfile
import csv
import datetime
import io
import json
import os
import os.path
import pathlib
import pprint
import pstats
import shlex
import shutil
import subprocess
import sys
import time
from runpy import run_module
from typing import Dict
import bitmath
import click
@ -43,6 +48,7 @@ from ._constants import (
OSXPHOTOS_EXPORT_DB,
OSXPHOTOS_URL,
POST_COMMAND_CATEGORIES,
PROFILE_SORT_KEYS,
SIDECAR_EXIFTOOL,
SIDECAR_JSON,
SIDECAR_XMP,
@ -69,7 +75,12 @@ from .pyrepl import embed_repl
from .queryoptions import QueryOptions
from .sqlgrep import sqlgrep
from .uti import get_preferred_uti_extension
from .utils import expand_and_validate_filepath, load_function, normalize_fs_path
from .utils import (
expand_and_validate_filepath,
list_directory,
load_function,
normalize_fs_path,
)
__all__ = [
"verbose_",
@ -121,6 +132,9 @@ __all__ = [
# set via --verbose/-V
VERBOSE = False
# used to show/hide hidden commands
OSXPHOTOS_HIDDEN = not bool(os.getenv("OSXPHOTOS_SHOW_HIDDEN", default=False))
# used by snap and diff commands
OSXPHOTOS_SNAPSHOT_DIR = "/private/tmp/osxphotos_snapshots"
@ -645,7 +659,9 @@ def QUERY_OPTIONS(f):
@click.group(context_settings=CTX_SETTINGS)
@DB_OPTION
@JSON_OPTION
@click.option("--debug", required=False, is_flag=True, default=False, hidden=True)
@click.option(
"--debug", required=False, is_flag=True, default=False, hidden=OSXPHOTOS_HIDDEN
)
@click.version_option(__version__, "--version", "-v")
@click.pass_context
def cli(ctx, db, json_, debug):
@ -1144,7 +1160,32 @@ def cli(ctx, db, json_, debug):
type=click.Path(),
)
@click.option(
"--beta", is_flag=True, default=False, hidden=True, help="Enable beta options."
"--beta",
is_flag=True,
default=False,
hidden=OSXPHOTOS_HIDDEN,
help="Enable beta options.",
)
@click.option(
"--profile",
is_flag=True,
default=False,
hidden=OSXPHOTOS_HIDDEN,
help="Run export with code profiler.",
)
@click.option(
"--profile-sort",
default=None,
hidden=OSXPHOTOS_HIDDEN,
multiple=True,
metavar="SORT_KEY",
type=click.Choice(
PROFILE_SORT_KEYS,
case_sensitive=True,
),
help="Sort profiler output by SORT_KEY as specified at https://docs.python.org/3/library/profile.html#pstats.Stats.sort_stats. "
f"Can be specified multiple times. Valid options are: {PROFILE_SORT_KEYS}. "
"Default = 'cumulative'.",
)
@DB_ARGUMENT
@click.argument("dest", nargs=1, type=click.Path(exists=True))
@ -1284,6 +1325,8 @@ def export(
preview,
preview_suffix,
preview_if_missing,
profile,
profile_sort,
):
"""Export photos from the Photos database.
Export path DEST is required.
@ -1297,6 +1340,24 @@ def export(
to modify this behavior.
"""
if profile:
click.echo("Profiling...")
profile_sort = profile_sort or ["cumulative"]
click.echo(f"Profile sort_stats order: {profile_sort}")
pr = cProfile.Profile()
pr.enable()
def at_exit():
pr.disable()
click.echo("Profiling completed")
s = io.StringIO()
pstats.Stats(pr, stream=s).strip_dirs().sort_stats(
*profile_sort
).print_stats()
click.echo(s.getvalue())
atexit.register(at_exit)
# NOTE: because of the way ConfigOptions works, Click options must not
# set defaults which are not None or False. If defaults need to be set
# do so below after load_config and save_config are handled.
@ -2046,6 +2107,18 @@ def export(
export_db.close()
def _export_with_profiler(args: Dict):
""" "Run export with cProfile"""
try:
args.pop("profile")
except KeyError:
pass
cProfile.runctx(
"_export(**args)", globals=globals(), locals=locals(), sort="tottime"
)
@cli.command()
@click.argument("topic", default=None, required=False, nargs=1)
@click.pass_context
@ -3457,8 +3530,7 @@ def cleanup_files(dest_path, files_to_keep, fileutil):
deleted_files = []
for p in pathlib.Path(dest_path).rglob("*"):
path = normalize_fs_path(str(p).lower())
if p.is_file() and path not in keepers:
if p.is_file() and normalize_fs_path(str(p).lower()) not in keepers:
verbose_(f"Deleting {p}")
fileutil.unlink(p)
deleted_files.append(str(p))
@ -4272,7 +4344,7 @@ def repl(ctx, cli_obj, db, emacs):
)
@cli.command(name="grep", hidden=True)
@cli.command(name="grep", hidden=OSXPHOTOS_HIDDEN)
@DB_OPTION
@click.pass_obj
@click.pass_context
@ -4318,7 +4390,7 @@ def grep(ctx, cli_obj, db, ignore_case, print_filename, pattern):
print(", ".join([table, column, row_id, value]))
@cli.command(hidden=True)
@cli.command(hidden=OSXPHOTOS_HIDDEN)
@DB_OPTION
@DB_ARGUMENT
@click.option(

48
osxphotos/utils.py
View File

@ -16,28 +16,29 @@ import sys
import unicodedata
import urllib.parse
from plistlib import load as plistload
from typing import Callable, Union
from typing import Callable, List, Union
import CoreFoundation
import objc
from Foundation import NSString
from Foundation import NSFileManager, NSString
from ._constants import UNICODE_FORMAT
__all__ = [
"noop",
"lineno",
"dd_to_dms_str",
"get_system_library_path",
"get_last_library_path",
"list_photo_libraries",
"normalize_fs_path",
"expand_and_validate_filepath",
"findfiles",
"normalize_unicode",
"get_last_library_path",
"get_system_library_path",
"increment_filename_with_count",
"increment_filename",
"expand_and_validate_filepath",
"lineno",
"list_directory",
"list_photo_libraries",
"load_function",
"noop",
"normalize_fs_path",
"normalize_unicode",
]
_DEBUG = False
@ -299,10 +300,21 @@ def findfiles(pattern, path_):
# paths need to be normalized for unicode as filesystem returns unicode in NFD form
pattern = normalize_fs_path(pattern)
rule = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
files = [normalize_fs_path(p) for p in os.listdir(path_)]
files = list_directory(path_)
return [name for name in files if rule.match(name)]
def list_directory(directory_path: str) -> List[str]:
"""List directory contents using NSFileManager"""
"""[[NSFileManager defaultManager] contentsOfDirectoryAtPath:@"directoryName" error:nil]"""
with objc.autorelease_pool():
manager = NSFileManager.defaultManager()
contents, error = manager.contentsOfDirectoryAtPath_error_(directory_path, None)
if error:
raise OSError(f"Error listing directory {directory_path}: {error}")
return [str(path) for path in contents]
def _open_sql_file(dbname):
"""opens sqlite file dbname in read-only mode
returns tuple of (connection, cursor)"""
@ -400,15 +412,15 @@ def increment_filename_with_count(
"""
dest = filepath if isinstance(filepath, pathlib.Path) else pathlib.Path(filepath)
dest_files = findfiles(f"{dest.stem}*", str(dest.parent))
dest_files = [normalize_fs_path(pathlib.Path(f).stem.lower()) for f in dest_files]
dest_new = dest.stem
if count:
dest_new = f"{dest.stem} ({count})"
while normalize_fs_path(dest_new.lower()) in dest_files:
dest_files = [pathlib.Path(f).stem.lower() for f in dest_files]
dest_new = f"{dest.stem} ({count})" if count else dest.stem
dest_new = normalize_fs_path(dest_new)
while dest_new.lower() in dest_files:
count += 1
dest_new = f"{dest.stem} ({count})"
dest_new = normalize_fs_path(f"{dest.stem} ({count})")
dest = dest.parent / f"{dest_new}{dest.suffix}"
return str(dest), count
return normalize_fs_path(str(dest)), count
def increment_filename(filepath: Union[str, pathlib.Path]) -> str: