Updated render_filepath_template to support multiple values

README.md

@@ -997,9 +997,14 @@ Render template string for photo.  none_str is used if template substitution res
 - `photo`: a [PhotoInfo](#photoinfo) object
 - `none_str`: optional str to use as substitution when template value is None and no default specified in the template string.  default is "_".
 
-Returns a tuple of (rendered, unmatched) where rendered is the rendered template string with all substitutions made and unmatched is a list of any strings that resembled a template substitution but did not match a known substitution. E.g. strings in the form "{foo}".
+Returns a tuple of (rendered, unmatched) where rendered is a list of rendered strings with all substitutions made and unmatched is a list of any strings that resembled a template substitution but did not match a known substitution. E.g. if template contained "{foo}", unmatched would be ["foo"].
+
+e.g. `render_filepath_template("{created.year}/{foo}", photo)` would return `("2020/{foo}",["foo"])`
+
+If you want to include "{" or "}" in the output, use "{{" or "}}"
+
+e.g. `render_filepath_template("{created.year}/{{foo}}", photo)` would return `("2020/{foo}",[])`
 
-e.g. `render_filepath_template("{created.year}/{foo}", photo)` would return `("2020/{foo}",["{foo}"])`
 
 | Substitution | Description |
 |--------------|-------------|

osxphotos/__main__.py

@@ -1518,6 +1518,7 @@ def export_photo(
         dest = create_path_by_date(dest, date_created)
     elif directory:
         dirname, unmatched = render_filepath_template(directory, photo)
+        dirname = dirname[0]
         if unmatched:
             click.echo(
                 f"Possible unmatched substitution in template: {unmatched}", err=True

osxphotos/_version.py

@@ -1,3 +1,3 @@
 """ version info """
 
-__version__ = "0.24.4"
+__version__ = "0.24.5"

osxphotos/template.py

@@ -1,10 +1,23 @@
+""" Custom template system for osxphotos """
+
+# Rolled my own template system because:
+# 1. Needed to handle multiple values (e.g. album, keyword)
+# 2. Needed to handle default values if template not found
+# 3. Didn't want user to need to know python (e.g. by using Mako which is
+#    already used elsewhere in this project)
+# 4. Couldn't figure out how to do #1 and #2 with str.format()
+#
+# This code isn't elegant but it seems to work well.  PRs gladly accepted.
+
 import datetime
 import pathlib
 import re
-from typing import Tuple  # pylint: disable=syntax-error
+from typing import Tuple, List  # pylint: disable=syntax-error
 
 from .photoinfo import PhotoInfo
+from ._constants import _UNKNOWN_PERSON
 
+# Permitted substitutions (each of these returns a single value or None)
 TEMPLATE_SUBSTITUTIONS = {
     "{name}": "Filename of the photo",
     "{original_name}": "Photo's original filename when imported to Photos",
@@ -39,9 +52,23 @@ TEMPLATE_SUBSTITUTIONS = {
     "{place.address.country_code}": "ISO country code of the postal address, e.g. 'US'",
 }
 
+# Permitted multi-value substitutions (each of these returns None or 1 or more values)
+TEMPLATE_SUBSTITUTIONS_MULTI_VALUED = {
+    "{album}": "Album photo is contained in",
+    "{keyword}": "Keywords assigned to photo",
+    "{person}": "Person / face in a photo",
+}
+
+# Just the multi-valued substitution names without the braces
+MULTI_VALUE_SUBSTITUTIONS = [
+    field.replace("{", "").replace("}", "")
+    for field in TEMPLATE_SUBSTITUTIONS_MULTI_VALUED.keys()
+]
+
 
 def get_template_value(lookup, photo):
-    """ lookup: value to find a match for
+    """ lookup template value (single-value template substitutions) for use in make_subst_function
+        lookup: value to find a match for
         photo: PhotoInfo object whose data will be used for value substitutions
         returns: either the matching template value (which may be None)
         raises: KeyError if no rule exists for lookup """
@@ -202,29 +229,24 @@ def get_template_value(lookup, photo):
     raise KeyError(f"No rule for processing {lookup}")
 
 
-def render_filepath_template(
+def render_filepath_template(template, photo, none_str="_"):
-    template: str, photo: PhotoInfo, none_str: str = "_"
+    """ render a filename or directory template 
-) -> Tuple[str, list]:
+        template: str template 
-    """ render a filename or directory template """
+        photo: PhotoInfo object
+        none_str: str to use default for None values, default is '_' """
 
+    # the rendering happens in two phases:
+    # phase 1: handle all the single-value template substitutions
+    #          results in a single string with all the template fields replaced
+    # phase 2: loop through all the multi-value template substitutions
+    #          could result in multiple strings
+    #          e.g. if template is "{album}/{person}" and there are 2 albums and 3 persons in the photo
+    #          there would be 6 possible renderings (2 albums x 3 persons)
+
+    # regex to find {template_field,optional_default} in strings
+    # for explanation of regex see https://regex101.com/r/4JJg42/1
     # pylint: disable=anomalous-backslash-in-string
-    regex = r"""(?<!\\)\{([^\\,}]+)(,{0,1}(([\w\-. ]+))?)\}"""
+    regex = r"(?<!\{)\{([^\\,}]+)(,{0,1}(([\w\-. ]+))?)(?=\}(?!\}))\}"
-
-    # pylint: disable=anomalous-backslash-in-string
-    unmatched_regex = r"(?<!\\)(\{[^\\,}]+\})"
-
-    # Explanation for regex:
-    # (?<!\\) Negative Lookbehind to skip escaped braces
-    #     assert regex following does not match "\" preceeding "{"
-    # \{ Match the opening brace
-    # 1st Capturing Group ([^\\,}]+)  Don't match "\", ",", or "}"
-    # 2nd Capturing Group (,?(([\w\-. ]+))?)
-    #     ,{0,1} optional ","
-    # 3rd Capturing Group (([\w\-. ]+))?
-    #     Matches the comma and any word characters after
-    # 4th Capturing Group ([\w\-. ]+)
-    #     Matches just the characters after the comma
-    # \} Matches the closing brace
 
     if type(template) is not str:
         raise TypeError(f"template must be type str, not {type(template)}")
@@ -232,14 +254,19 @@ def render_filepath_template(
     if type(photo) is not PhotoInfo:
         raise TypeError(f"photo must be type osxphotos.PhotoInfo, not {type(photo)}")
 
-    def make_subst_function(photo, none_str):
+    def make_subst_function(photo, none_str, get_func=get_template_value):
-        """ returns: substitution function for use in re.sub """
+        """ returns: substitution function for use in re.sub 
+            photo: a PhotoInfo object
+            none_str: value to use if substitution lookup is None and no default provided
+            get_func: function that gets the substitution value for a given template field
+                      default is get_template_value which handles the single-value fields """
+
         # closure to capture photo, none_str in subst
         def subst(matchobj):
             groups = len(matchobj.groups())
             if groups == 4:
                 try:
-                    val = get_template_value(matchobj.group(1), photo)
+                    val = get_func(matchobj.group(1), photo)
                 except KeyError:
                     return matchobj.group(0)
 
@@ -261,14 +288,92 @@ def render_filepath_template(
     # do the replacements
     rendered = re.sub(regex, subst_func, template)
 
-    # find any {words} that weren't replaced
+    # do multi-valued placements
-    unmatched = re.findall(unmatched_regex, rendered)
+    # start with the single string from phase 1 above then loop through all
+    # multi-valued fields and all values for each of those fields
+    # rendered_strings will be updated as each field is processed
+    # for example: if two albums, two keywords, and one person and template is:
+    # "{created.year}/{album}/{keyword}/{person}"
+    # rendered strings would do the following:
+    # start (created.year filled in phase 1)
+    #   ['2011/{album}/{keyword}/{person}']
+    # after processing albums:
+    #   ['2011/Album1/{keyword}/{person}',
+    #    '2011/Album2/{keyword}/{person}',]
+    # after processing keywords:
+    #   ['2011/Album1/keyword1/{person}',
+    #    '2011/Album1/keyword2/{person}',
+    #    '2011/Album2/keyword1/{person}',
+    #    '2011/Album2/keyword2/{person}',]
+    # after processing person:
+    #   ['2011/Album1/keyword1/person1',
+    #    '2011/Album1/keyword2/person1',
+    #    '2011/Album2/keyword1/person1',
+    #    '2011/Album2/keyword2/person1',]
+
+    rendered_strings = [rendered]
+    for field in MULTI_VALUE_SUBSTITUTIONS:
+        if field == "album":
+            values = photo.albums
+        elif field == "keyword":
+            values = photo.keywords
+        elif field == "person":
+            values = photo.persons
+            # remove any _UNKNOWN_PERSON values
+            try:
+                values.remove(_UNKNOWN_PERSON)
+            except:
+                pass
+
+        else:
+            raise ValueError(f"Unhandleded template value: {field}")
+
+        # If no values, insert None so code below will substite none_str for None
+        values = values or [None]
+
+        # Build a regex that matches only the field being processed
+        re_str = r"(?<!\\)\{(" + field + r")(,{0,1}(([\w\-. ]+))?)\}"
+        regex_multi = re.compile(re_str)
+        new_strings = []  # holds each of the new rendered_strings
+        for str_template in rendered_strings:
+            for val in values:
+
+                def get_template_value_multi(lookup_value, photo):
+                    """ Closure passed to make_subst_function get_func 
+                        Capture val and field in the closure 
+                        Allows make_subst_function to be re-used w/o modification """
+                    if lookup_value == field:
+                        return val
+                    else:
+                        raise KeyError(f"Unexpected value: {lookup_value}")
+
+                subst = make_subst_function(
+                    photo, none_str, get_func=get_template_value_multi
+                )
+                new_string = regex_multi.sub(subst, str_template)
+                new_strings.append(new_string)
+
+        # update rendered_strings for the next field to process
+        rendered_strings = new_strings
+
+    # find any {fields} that weren't replaced
+    unmatched = []
+    for rendered_str in rendered_strings:
+        unmatched.extend(
+            [
+                no_match[0]
+                for no_match in re.findall(regex, rendered_str)
+                if no_match[0] not in unmatched
+            ]
+        )
 
     # fix any escaped curly braces
-    rendered = re.sub(r"\\{", "{", rendered)
+    rendered_strings = [
-    rendered = re.sub(r"\\}", "}", rendered)
+        rendered_str.replace("{{", "{").replace("}}", "}")
+        for rendered_str in rendered_strings
+    ]
 
-    return rendered, unmatched
+    return rendered_strings, unmatched
 
 
 class DateTimeFormatter:

tests/test_cli.py

@@ -408,7 +408,7 @@ def test_export_directory_template_3():
             ],
         )
         assert result.exit_code == 0
-        assert "Possible unmatched substitution in template: ['{foo}']" in result.output
+        assert "Possible unmatched substitution in template: ['foo']" in result.output
         workdir = os.getcwd()
         for filepath in CLI_EXPORTED_DIRECTORY_TEMPLATE_FILENAMES3:
             assert os.path.isfile(os.path.join(workdir, filepath))
@@ -453,6 +453,7 @@ def test_place_13():
         assert len(json_got) == 1  # single element
         assert json_got[0]["uuid"] == "2L6X2hv3ROWRSCU3WRRAGQ"
 
+
 def test_no_place_13():
     # test --no-place on 10.13
     import json
@@ -466,8 +467,7 @@ def test_no_place_13():
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
         result = runner.invoke(
-            query,
+            query, [os.path.join(cwd, PLACES_PHOTOS_DB_13), "--json", "--no-place"]
-            [os.path.join(cwd, PLACES_PHOTOS_DB_13), "--json", "--no-place"],
         )
         assert result.exit_code == 0
         json_got = json.loads(result.output)
@@ -498,6 +498,7 @@ def test_place_15_1():
         assert len(json_got) == 1  # single element
         assert json_got[0]["uuid"] == "128FB4C6-0B16-4E7D-9108-FB2E90DA1546"
 
+
 def test_place_15_2():
     # test --place on 10.15
     import json
@@ -518,7 +519,7 @@ def test_place_15_2():
         json_got = json.loads(result.output)
 
         assert len(json_got) == 2  # single element
-        uuid = [json_got[x]["uuid"] for x in (0,1)]
+        uuid = [json_got[x]["uuid"] for x in (0, 1)]
         assert "128FB4C6-0B16-4E7D-9108-FB2E90DA1546" in uuid
         assert "FF7AFE2C-49B0-4C9B-B0D7-7E1F8B8F2F0C" in uuid
 
@@ -536,11 +537,10 @@ def test_no_place_15():
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
         result = runner.invoke(
-            query,
+            query, [os.path.join(cwd, PLACES_PHOTOS_DB), "--json", "--no-place"]
-            [os.path.join(cwd, PLACES_PHOTOS_DB), "--json", "--no-place"],
         )
         assert result.exit_code == 0
         json_got = json.loads(result.output)
 
         assert len(json_got) == 1  # single element
-        assert json_got[0]["uuid"] == "A9B73E13-A6F2-4915-8D67-7213B39BAE9F"
+        assert json_got[0]["uuid"] == "A9B73E13-A6F2-4915-8D67-7213B39BAE9F"

tests/test_template.py

@@ -1,10 +1,14 @@
 """ Test template.py """
 import pytest
 
-PHOTOS_DB = "./tests/Test-Places-Catalina-10_15_1.photoslibrary/database/photos.db"
+PHOTOS_DB_1 = "./tests/Test-Places-Catalina-10_15_1.photoslibrary/database/photos.db"
-
+PHOTOS_DB_2 = "./tests/Test-10.15.1.photoslibrary/database/photos.db"
-UUID_DICT = {"place_dc": "128FB4C6-0B16-4E7D-9108-FB2E90DA1546"}
+UUID_DICT = {
-
+    "place_dc": "128FB4C6-0B16-4E7D-9108-FB2E90DA1546",
+    "1_1_2": "1EB2B765-0765-43BA-A90C-0D0580E6172C",
+    "2_1_1": "D79B8D77-BFFC-460B-9312-034F2877D35B",
+    "0_2_0": "6191423D-8DB8-4D4C-92BE-9BBBA308AAC4",
+}
 
 TEMPLATE_VALUES = {
     "{name}": "128FB4C6-0B16-4E7D-9108-FB2E90DA1546",
@@ -51,7 +55,7 @@ def test_lookup():
         TEMPLATE_SUBSTITUTIONS,
     )
 
-    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
     photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
 
     for subst in TEMPLATE_SUBSTITUTIONS:
@@ -67,12 +71,12 @@ def test_subst():
     from osxphotos.template import render_filepath_template
 
     locale.setlocale(locale.LC_ALL, "en_US")
-    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
     photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
 
     for template in TEMPLATE_VALUES:
         rendered, _ = render_filepath_template(template, photo)
-        assert rendered == TEMPLATE_VALUES[template]
+        assert rendered[0] == TEMPLATE_VALUES[template]
 
 
 def test_subst_default_val():
@@ -82,12 +86,12 @@ def test_subst_default_val():
     from osxphotos.template import render_filepath_template
 
     locale.setlocale(locale.LC_ALL, "en_US")
-    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
     photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
 
     template = "{place.name.area_of_interest,UNKNOWN}"
     rendered, _ = render_filepath_template(template, photo)
-    assert rendered == "UNKNOWN"
+    assert rendered[0] == "UNKNOWN"
 
 
 def test_subst_default_val_2():
@@ -97,12 +101,12 @@ def test_subst_default_val_2():
     from osxphotos.template import render_filepath_template
 
     locale.setlocale(locale.LC_ALL, "en_US")
-    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
     photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
 
     template = "{place.name.area_of_interest,}"
     rendered, _ = render_filepath_template(template, photo)
-    assert rendered == "_"
+    assert rendered[0] == "_"
 
 
 def test_subst_unknown_val():
@@ -112,10 +116,149 @@ def test_subst_unknown_val():
     from osxphotos.template import render_filepath_template
 
     locale.setlocale(locale.LC_ALL, "en_US")
-    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB)
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
     photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
 
     template = "{created.year}/{foo}"
     rendered, unknown = render_filepath_template(template, photo)
-    assert rendered == "2020/{foo}"
+    assert rendered[0] == "2020/{foo}"
-    assert unknown == ["{foo}"]
+    assert unknown == ["foo"]
+
+    template = "{place.name.area_of_interest,}"
+    rendered, _ = render_filepath_template(template, photo)
+    assert rendered[0] == "_"
+
+
+def test_subst_double_brace():
+    """ Test substitution with double brace {{ which should be ignored """
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
+    photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
+
+    template = "{created.year}/{{foo}}"
+    rendered, unknown = render_filepath_template(template, photo)
+    assert rendered[0] == "2020/{foo}"
+    assert not unknown
+
+
+def test_subst_unknown_val_with_default():
+    """ Test substitution with unknown value specified """
+    import locale
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    locale.setlocale(locale.LC_ALL, "en_US")
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_1)
+    photo = photosdb.photos(uuid=[UUID_DICT["place_dc"]])[0]
+
+    template = "{created.year}/{foo,bar}"
+    rendered, unknown = render_filepath_template(template, photo)
+    assert rendered[0] == "2020/{foo,bar}"
+    assert unknown == ["foo"]
+
+
+def test_subst_multi_1_1_2():
+    """ Test that substitutions are correct """
+    # one album, one keyword, two persons
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    photo = photosdb.photos(uuid=[UUID_DICT["1_1_2"]])[0]
+
+    template = "{created.year}/{album}/{keyword}/{person}"
+    expected = ["2018/Pumpkin Farm/Kids/Katie", "2018/Pumpkin Farm/Kids/Suzy"]
+    rendered, _ = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+
+
+def test_subst_multi_2_1_1():
+    """ Test that substitutions are correct """
+    # 2 albums, 1 keyword, 1 person
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    # one album, one keyword, two persons
+    photo = photosdb.photos(uuid=[UUID_DICT["2_1_1"]])[0]
+
+    template = "{created.year}/{album}/{keyword}/{person}"
+    expected = ["2018/Pumpkin Farm/Kids/Katie", "2018/Test Album/Kids/Katie"]
+    rendered, _ = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+
+
+def test_subst_multi_0_2_0():
+    """ Test that substitutions are correct """
+    # 0 albums, 2 keywords, 0 persons
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    # one album, one keyword, two persons
+    photo = photosdb.photos(uuid=[UUID_DICT["0_2_0"]])[0]
+
+    template = "{created.year}/{album}/{keyword}/{person}"
+    expected = ["2019/_/wedding/_", "2019/_/flowers/_"]
+    rendered, _ = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+
+
+def test_subst_multi_0_2_0_default_val():
+    """ Test that substitutions are correct """
+    # 0 albums, 2 keywords, 0 persons, default vals provided
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    # one album, one keyword, two persons
+    photo = photosdb.photos(uuid=[UUID_DICT["0_2_0"]])[0]
+
+    template = "{created.year}/{album,NOALBUM}/{keyword,NOKEYWORD}/{person,NOPERSON}"
+    expected = ["2019/NOALBUM/wedding/NOPERSON", "2019/NOALBUM/flowers/NOPERSON"]
+    rendered, _ = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+
+
+def test_subst_multi_0_2_0_default_val_unknown_val():
+    """ Test that substitutions are correct """
+    # 0 albums, 2 keywords, 0 persons, default vals provided, unknown val in template
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    # one album, one keyword, two persons
+    photo = photosdb.photos(uuid=[UUID_DICT["0_2_0"]])[0]
+
+    template = (
+        "{created.year}/{album,NOALBUM}/{keyword,NOKEYWORD}/{person}/{foo}/{{baz}}"
+    )
+    expected = [
+        "2019/NOALBUM/wedding/_/{foo}/{baz}",
+        "2019/NOALBUM/flowers/_/{foo}/{baz}",
+    ]
+    rendered, unknown = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+    assert unknown == ["foo"]
+
+
+def test_subst_multi_0_2_0_default_val_unknown_val_2():
+    """ Test that substitutions are correct """
+    # 0 albums, 2 keywords, 0 persons, default vals provided, unknown val in template
+    import osxphotos
+    from osxphotos.template import render_filepath_template
+
+    photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB_2)
+    # one album, one keyword, two persons
+    photo = photosdb.photos(uuid=[UUID_DICT["0_2_0"]])[0]
+
+    template = "{created.year}/{album,NOALBUM}/{keyword,NOKEYWORD}/{person}/{foo,bar}/{{baz,bar}}"
+    expected = [
+        "2019/NOALBUM/wedding/_/{foo,bar}/{baz,bar}",
+        "2019/NOALBUM/flowers/_/{foo,bar}/{baz,bar}",
+    ]
+    rendered, unknown = render_filepath_template(template, photo)
+    assert sorted(rendered) == sorted(expected)
+    assert unknown == ["foo"]