From fc416ea0b746413d2ac2f1dd2f1bc200227013ed Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sat, 8 Aug 2020 21:01:53 -0700 Subject: [PATCH] Fixed from_date and to_date to be timezone aware, closes #193 --- osxphotos/__main__.py | 32 ++++++++++---- osxphotos/_version.py | 2 +- osxphotos/datetime_utils.py | 57 +++++++++++++++++++++++++ osxphotos/photoinfo/photoinfo.py | 8 +--- osxphotos/photosdb/photosdb.py | 73 ++++++++++++++++++++++---------- tests/test_10_12_6.py | 7 --- tests/test_catalina_10_15_1.py | 7 --- tests/test_catalina_10_15_4.py | 7 --- tests/test_catalina_10_15_5.py | 7 --- tests/test_cli.py | 65 +++++++++++++++++++++++++++- tests/test_empty_library_4_0.py | 7 --- tests/test_highsierra.py | 7 --- tests/test_mojave_10_14_5.py | 7 --- tests/test_mojave_10_14_6.py | 9 +--- 14 files changed, 199 insertions(+), 96 deletions(-) create mode 100644 osxphotos/datetime_utils.py diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index 7a63834a..3d9e2052 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -77,6 +77,20 @@ def get_photos_db(*db_options): return None +class DateTimeISO8601(click.ParamType): + + name = "DATETIME" + + def convert(self, value, param, ctx): + try: + return datetime.datetime.fromisoformat(value) + except: + self.fail( + f"Invalid value for --{param.name}: invalid datetime format {value}. " + "Valid format: YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]" + ) + + # Click CLI object & context settings class CLI_Obj: def __init__(self, db=None, json=False, debug=False): @@ -305,7 +319,7 @@ def query_options(f): multiple=False, help="Search for photos with UUID(s) loaded from FILE. " "Format is a single UUID per line. Lines preceeded with # are ignored.", - type=click.Path(exists=True) + type=click.Path(exists=True), ), o( "--title", @@ -454,13 +468,13 @@ def query_options(f): ), o( "--from-date", - help="Search by start item date, e.g. 2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 w/o TZ).", - type=click.DateTime(), + help="Search by start item date, e.g. 2000-01-12T12:00:00, 2001-01-12T12:00:00-07:00, or 2000-12-31 (ISO 8601).", + type=DateTimeISO8601(), ), o( "--to-date", - help="Search by end item date, e.g. 2000-01-12T12:00:00 or 2000-12-31 (ISO 8601 w/o TZ).", - type=click.DateTime(), + help="Search by end item date, e.g. 2000-01-12T12:00:00, 2001-01-12T12:00:00-07:00, or 2000-12-31 (ISO 8601).", + type=DateTimeISO8601(), ), ] for o in options[::-1]: @@ -1011,7 +1025,7 @@ def query( # load UUIDs if necessary and append to any uuids passed with --uuid if uuid_from_file: - uuid_list = list(uuid) # Click option is a tuple + uuid_list = list(uuid) # Click option is a tuple uuid_list.extend(load_uuid_from_file(uuid_from_file)) uuid = tuple(uuid_list) @@ -1401,7 +1415,7 @@ def export( # load UUIDs if necessary and append to any uuids passed with --uuid if uuid_from_file: - uuid_list = list(uuid) # Click option is a tuple + uuid_list = list(uuid) # Click option is a tuple uuid_list.extend(load_uuid_from_file(uuid_from_file)) uuid = tuple(uuid_list) @@ -2363,6 +2377,7 @@ def find_files_in_branch(pathname, filename): return files + def load_uuid_from_file(filename): """ Load UUIDs from file. Does not validate UUIDs. Format is 1 UUID per line, any line beginning with # is ignored. @@ -2377,7 +2392,7 @@ def load_uuid_from_file(filename): Raises: FileNotFoundError if file does not exist """ - + if not pathlib.Path(filename).is_file(): raise FileNotFoundError(f"Could not find file {filename}") @@ -2389,5 +2404,6 @@ def load_uuid_from_file(filename): uuid.append(line) return uuid + if __name__ == "__main__": cli() # pylint: disable=no-value-for-parameter diff --git a/osxphotos/_version.py b/osxphotos/_version.py index ad81d483..ca5b4390 100644 --- a/osxphotos/_version.py +++ b/osxphotos/_version.py @@ -1,3 +1,3 @@ """ version info """ -__version__ = "0.31.1" +__version__ = "0.31.2" diff --git a/osxphotos/datetime_utils.py b/osxphotos/datetime_utils.py new file mode 100644 index 00000000..57267715 --- /dev/null +++ b/osxphotos/datetime_utils.py @@ -0,0 +1,57 @@ +""" datetime utilities """ + +import datetime + +def get_local_tz(): + """ return local timezone as datetime.timezone tzinfo """ + local_tz = ( + datetime.datetime.now(datetime.timezone(datetime.timedelta(0))) + .astimezone() + .tzinfo + ) + return local_tz + + +def datetime_remove_tz(dt): + """ remove timezone from a datetime.datetime object + dt: datetime.datetime object with tzinfo + returns: dt without any timezone info (naive datetime object) """ + + if type(dt) != datetime.datetime: + raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}") + + dt_new = dt.replace(tzinfo=None) + return dt_new + + +def datetime_has_tz(dt): + """ return True if datetime dt has tzinfo else False + dt: datetime.datetime + returns True if dt is timezone aware, else False """ + + if type(dt) != datetime.datetime: + raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}") + + if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None: + return True + return False + + +def datetime_naive_to_local(dt): + """ convert naive (timezone unaware) datetime.datetime + to aware timezone in local timezone + dt: datetime.datetime without timezone + returns: datetime.datetime with local timezone """ + + if type(dt) != datetime.datetime: + raise TypeError(f"dt must be type datetime.datetime, not {type(dt)}") + + if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None: + # has timezone info + raise ValueError( + "dt must be naive/timezone unaware: " + f"{dt} has tzinfo {dt.tzinfo} and offset {dt.tizinfo.utcoffset(dt)}" + ) + + dt_local = dt.replace(tzinfo=get_local_tz()) + return dt_local diff --git a/osxphotos/photoinfo/photoinfo.py b/osxphotos/photoinfo/photoinfo.py index ed17c312..d2b9707a 100644 --- a/osxphotos/photoinfo/photoinfo.py +++ b/osxphotos/photoinfo/photoinfo.py @@ -87,12 +87,8 @@ class PhotoInfo: @property def date(self): """ image creation date as timezone aware datetime object """ - imagedate = self._info["imageDate"] - seconds = self._info["imageTimeZoneOffsetSeconds"] or 0 - delta = timedelta(seconds=seconds) - tz = timezone(delta) - return imagedate.astimezone(tz=tz) - + return self._info["imageDate"] + @property def date_modified(self): """ image modification date as timezone aware datetime object diff --git a/osxphotos/photosdb/photosdb.py b/osxphotos/photosdb/photosdb.py index 3b8dc63c..cb2cbe05 100644 --- a/osxphotos/photosdb/photosdb.py +++ b/osxphotos/photosdb/photosdb.py @@ -11,7 +11,7 @@ import platform import sqlite3 import sys import tempfile -from datetime import datetime +from datetime import datetime, timedelta, timezone from pprint import pformat from shutil import copyfile @@ -34,6 +34,7 @@ from .._constants import ( ) from .._version import __version__ from ..albuminfo import AlbumInfo, FolderInfo +from ..datetime_utils import datetime_has_tz, datetime_naive_to_local from ..personinfo import PersonInfo from ..photoinfo import PhotoInfo from ..utils import ( @@ -952,15 +953,23 @@ class PhotosDB: except TypeError: self._dbphotos[uuid]["lastmodifieddate"] = None + self._dbphotos[uuid]["imageTimeZoneOffsetSeconds"] = row[9] + try: - self._dbphotos[uuid]["imageDate"] = datetime.fromtimestamp(row[5] + td) + imagedate = datetime.fromtimestamp(row[5] + td) + seconds = self._dbphotos[uuid]["imageTimeZoneOffsetSeconds"] or 0 + delta = timedelta(seconds=seconds) + tz = timezone(delta) + self._dbphotos[uuid]["imageDate"] = imagedate.astimezone(tz=tz) except ValueError: - self._dbphotos[uuid]["imageDate"] = datetime(1970, 1, 1) + # sometimes imageDate is invalid so use 1 Jan 1970 in UTC as image date + imagedate = datetime(1970, 1, 1) + tz = timezone(timedelta(0)) + self._dbphotos[uuid]["imageDate"] = imagedate.astimezone(tz=tz) self._dbphotos[uuid]["mainRating"] = row[6] self._dbphotos[uuid]["hasAdjustments"] = row[7] self._dbphotos[uuid]["hasKeywords"] = row[8] - self._dbphotos[uuid]["imageTimeZoneOffsetSeconds"] = row[9] self._dbphotos[uuid]["volumeId"] = row[10] self._dbphotos[uuid]["imagePath"] = row[11] self._dbphotos[uuid]["extendedDescription"] = row[12] @@ -1329,7 +1338,7 @@ class PhotosDB: # process faces self._process_faceinfo() - + # add faces and keywords to photo data for uuid in self._dbphotos: # keywords @@ -1788,12 +1797,20 @@ class PhotosDB: except TypeError: info["lastmodifieddate"] = None - try: - info["imageDate"] = datetime.fromtimestamp(row[5] + td) - except ValueError: - info["imageDate"] = datetime(1970, 1, 1) - info["imageTimeZoneOffsetSeconds"] = row[6] + + try: + imagedate = datetime.fromtimestamp(row[5] + td) + seconds = info["imageTimeZoneOffsetSeconds"] or 0 + delta = timedelta(seconds=seconds) + tz = timezone(delta) + info["imageDate"] = imagedate.astimezone(tz=tz) + except ValueError: + # sometimes imageDate is invalid so use 1 Jan 1970 in UTC as image date + imagedate = datetime(1970, 1, 1) + tz = timezone(timedelta(0)) + info["imageDate"] = imagedate.astimezone(tz=tz) + info["hidden"] = row[9] info["favorite"] = row[10] info["originalFilename"] = row[3] @@ -2135,7 +2152,7 @@ class PhotosDB: # process face info self._process_faceinfo() - + # process search info self._process_searchinfo() @@ -2436,23 +2453,29 @@ class PhotosDB: to_date=None, intrash=False, ): - """ - Return a list of PhotoInfo objects + """ Return a list of PhotoInfo objects If called with no args, returns the entire database of photos If called with args, returns photos matching the args (e.g. keywords, persons, etc.) If more than one arg, returns photos matching all the criteria (e.g. keywords AND persons) If more than one keyword, uuid, persons, albums is passed, they are treated as "OR" criteria e.g. keywords=["wedding","vacation"] returns photos matching either keyword - keywords: list of keywords to search for - uuid: list of UUIDs to search for - persons: list of persons to search for - albums: list of album names to search for - images: if True, returns image files, if False, does not return images; default is True - movies: if True, returns movie files, if False, does not return movies; default is True - from_date: return photos with creation date >= from_date (datetime.datetime object, default None) - to_date: return photos with creation date <= to_date (datetime.datetime object, default None) - intrash: if True, returns only images in "Recently deleted items" folder, - if False returns only photos that aren't deleted; default is False + from_date and to_date may be either naive or timezone-aware datetime.datetime objects. + If naive, timezone will be assumed to be local timezone. + + Args: + keywords: list of keywords to search for + uuid: list of UUIDs to search for + persons: list of persons to search for + albums: list of album names to search for + images: if True, returns image files, if False, does not return images; default is True + movies: if True, returns movie files, if False, does not return movies; default is True + from_date: return photos with creation date >= from_date (datetime.datetime object, default None) + to_date: return photos with creation date <= to_date (datetime.datetime object, default None) + intrash: if True, returns only images in "Recently deleted items" folder, + if False returns only photos that aren't deleted; default is False + + Returns: + list of PhotoInfo objects """ # implementation is a bit kludgy but it works @@ -2528,6 +2551,8 @@ class PhotosDB: if from_date or to_date: # sourcery off dsel = self._dbphotos if from_date: + if not datetime_has_tz(from_date): + from_date = datetime_naive_to_local(from_date) dsel = { k: v for k, v in dsel.items() if v["imageDate"] >= from_date } @@ -2535,6 +2560,8 @@ class PhotosDB: f"Found %i items with from_date {from_date}" % len(dsel) ) if to_date: + if not datetime_has_tz(to_date): + to_date = datetime_naive_to_local(to_date) dsel = {k: v for k, v in dsel.items() if v["imageDate"] <= to_date} logging.debug(f"Found %i items with to_date {to_date}" % len(dsel)) photos_sets.append(set(dsel.keys())) diff --git a/tests/test_10_12_6.py b/tests/test_10_12_6.py index ffba6898..7c8eab24 100644 --- a/tests/test_10_12_6.py +++ b/tests/test_10_12_6.py @@ -46,13 +46,6 @@ def test_db_version(): assert photosdb.db_version == "2622" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_catalina_10_15_1.py b/tests/test_catalina_10_15_1.py index 641e76d9..8382a470 100644 --- a/tests/test_catalina_10_15_1.py +++ b/tests/test_catalina_10_15_1.py @@ -130,13 +130,6 @@ def test_db_version(): assert photosdb.db_version == "6000" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_catalina_10_15_4.py b/tests/test_catalina_10_15_4.py index 820febc3..4f428b70 100644 --- a/tests/test_catalina_10_15_4.py +++ b/tests/test_catalina_10_15_4.py @@ -138,13 +138,6 @@ def test_db_version(): assert photosdb.db_version == "6000" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_catalina_10_15_5.py b/tests/test_catalina_10_15_5.py index efbb2214..bc5dd94a 100644 --- a/tests/test_catalina_10_15_5.py +++ b/tests/test_catalina_10_15_5.py @@ -164,13 +164,6 @@ def test_db_version(): assert photosdb.db_version == "6000" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_cli.py b/tests/test_cli.py index 8e7cd66c..83ddffba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -697,13 +697,18 @@ def test_export_edited_suffix(): assert sorted(files) == sorted(CLI_EXPORT_FILENAMES_EDITED_SUFFIX) -def test_query_date(): +def test_query_date_1(): + """ Test --from-date and --to-date """ import json import osxphotos import os import os.path + import time from osxphotos.__main__ import query + os.environ['TZ'] = "US/Pacific" + time.tzset() + runner = CliRunner() cwd = os.getcwd() result = runner.invoke( @@ -721,6 +726,64 @@ def test_query_date(): json_got = json.loads(result.output) assert len(json_got) == 4 +def test_query_date_2(): + """ Test --from-date and --to-date """ + import json + import osxphotos + import os + import os.path + import time + from osxphotos.__main__ import query + + os.environ['TZ'] = "Asia/Jerusalem" + time.tzset() + + runner = CliRunner() + cwd = os.getcwd() + result = runner.invoke( + query, + [ + "--json", + "--db", + os.path.join(cwd, CLI_PHOTOS_DB), + "--from-date=2018-09-28", + "--to-date=2018-09-28T23:00:00", + ], + ) + assert result.exit_code == 0 + + json_got = json.loads(result.output) + assert len(json_got) == 2 + +def test_query_date_timezone(): + """ Test --from-date, --to-date with ISO 8601 timezone """ + import json + import osxphotos + import os + import os.path + import time + from osxphotos.__main__ import query + + os.environ['TZ'] = "US/Pacific" + time.tzset() + + runner = CliRunner() + cwd = os.getcwd() + result = runner.invoke( + query, + [ + "--json", + "--db", + os.path.join(cwd, CLI_PHOTOS_DB), + "--from-date=2018-09-28T00:00:00-07:00", + "--to-date=2018-09-28T23:00:00-07:00", + ], + ) + assert result.exit_code == 0 + + json_got = json.loads(result.output) + assert len(json_got) == 4 + def test_query_keyword_1(): """Test query --keyword """ diff --git a/tests/test_empty_library_4_0.py b/tests/test_empty_library_4_0.py index 3057e0b8..7121e11d 100644 --- a/tests/test_empty_library_4_0.py +++ b/tests/test_empty_library_4_0.py @@ -29,13 +29,6 @@ def test_db_version(): assert photosdb.db_version == "4025" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_highsierra.py b/tests/test_highsierra.py index 789ef3ba..fbb7af1a 100644 --- a/tests/test_highsierra.py +++ b/tests/test_highsierra.py @@ -46,13 +46,6 @@ def test_db_version(): # assert photosdb.db_version in osxphotos._TESTED_DB_VERSIONS -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_mojave_10_14_5.py b/tests/test_mojave_10_14_5.py index 31a24aec..0bcaf575 100644 --- a/tests/test_mojave_10_14_5.py +++ b/tests/test_mojave_10_14_5.py @@ -47,13 +47,6 @@ def test_db_version(): assert photosdb.db_version == "4016" -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections diff --git a/tests/test_mojave_10_14_6.py b/tests/test_mojave_10_14_6.py index c47c6632..508af580 100644 --- a/tests/test_mojave_10_14_6.py +++ b/tests/test_mojave_10_14_6.py @@ -1,4 +1,3 @@ -import pytest from osxphotos._constants import _UNKNOWN_PERSON @@ -82,13 +81,6 @@ def test_db_len(): assert len(photosdb) == PHOTOS_DB_LEN -def test_os_version(): - import osxphotos - - (_, major, _) = osxphotos.utils._get_os_version() - assert major in osxphotos._constants._TESTED_OS_VERSIONS - - def test_persons(): import osxphotos import collections @@ -272,6 +264,7 @@ def test_not_hidden(): def test_location_1(): # test photo with lat/lon info import osxphotos + import pytest photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB) photos = photosdb.photos(uuid=["3Jn73XpSQQCluzRBMWRsMA"])