From b5a9794f6bff5683fd42a22197454940e4d7ba88 Mon Sep 17 00:00:00 2001 From: Rhet Turnbull Date: Sun, 11 Oct 2020 08:59:49 -0700 Subject: [PATCH] Added israw, tests for Big Sur --- README.md | 15 ++-- osxphotos/__main__.py | 18 ++-- osxphotos/photoinfo/photoinfo.py | 14 +++- osxphotos/photosdb/photosdb.py | 55 ++++++------- .../database/photos.db | Bin 2056192 -> 2056192 bytes .../PhotoAnalysisServicePreferences.plist | 2 +- .../PhotosGraph/photosgraph.graphdb | Bin 94208 -> 94208 bytes .../PhotosGraph/photosgraph.graphdb-shm | Bin 32768 -> 32768 bytes .../resources/recovery/Info.plist | 2 +- tests/test_bigsur_10_16_0.py | 77 +++++++++++++++++- tests/test_mojave_10_14_6.py | 2 +- 11 files changed, 133 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 93cf986b..d3609051 100644 --- a/README.md +++ b/README.md @@ -1092,13 +1092,16 @@ Returns the absolute path to the edited photo on disk as a string. If the photo **Note**: will also return None if the edited photo is missing on disk. #### `path_raw` -Returns the absolute path to the associated RAW photo on disk as a string, if photo is part of a RAW+JPEG pair, otherwise returns None. +Returns the absolute path to the associated raw photo on disk as a string, if photo is part of a RAW+JPEG pair, otherwise returns None. #### `has_raw` -Returns True if photo has an associated RAW image, otherwise False. (e.g. Photo is a RAW+JPEG pair.) +Returns True if photo has an associated raw image, otherwise False. (e.g. Photo is a RAW+JPEG pair). See also [is_raw](#israw). + +#### `israw` +Returns True if photo is a raw image. E.g. it was imported as a single raw image, not part of a RAW+JPEG pair. See also [has_raw](#has_raw). #### `raw_original` -Returns True if associated RAW image and the RAW image is selected in Photos via "Use RAW as Original", otherwise returns False. Not implemented on Photos version < 5; returns None on Photos 4 and below. +Returns True if associated raw image and the raw image is selected in Photos via "Use RAW as Original", otherwise returns False. #### `height` Returns height of the photo in pixels. If image has been edited, returns height of the edited image, otherwise returns height of the original image. See also [original_height](#original_height). @@ -1174,7 +1177,7 @@ Returns Uniform Type Identifier (UTI) for the original image, for example: 'publ Returns Uniform Type Identifier (UTI) for the edited image, for example: 'public.jpeg'. Returns None if the photo does not have adjustments. #### `uti_raw` -Returns Uniform Type Identifier (UTI) for the associated RAW image, if there is one; for example, 'com.canon.cr2-raw-image'. If the image is RAW but not part of a RAW+JPEG pair, `uti_raw` returns None. In this case, use `uti`, or `uti_original`. +Returns Uniform Type Identifier (UTI) for the associated raw image, if there is one; for example, 'com.canon.cr2-raw-image'. If the image is raw but not part of a RAW+JPEG pair, `uti_raw` returns None. In this case, use `uti`, or `uti_original`. See also [has_raw](#has_raw). #### `burst` Returns True if photos is a burst image (e.g. part of a set of burst images), otherwise False. @@ -1909,10 +1912,10 @@ Thank-you to the following people who have contributed to improving osxphotos! ## Known Bugs -My goal is make osxphotos as reliable and comprehensive as possible. The test suite currently has over 600 tests--but there are still some [bugs](https://github.com/RhetTbull/osxphotos/issues?q=is%3Aissue+is%3Aopen+label%3Abug) or incomplete features lurking. If you find bugs please open an [issue](https://github.com/RhetTbull/osxphotos/issues). Notable issues include: +My goal is make osxphotos as reliable and comprehensive as possible. The test suite currently has over 800 tests--but there are still some [bugs](https://github.com/RhetTbull/osxphotos/issues?q=is%3Aissue+is%3Aopen+label%3Abug) or incomplete features lurking. If you find bugs please open an [issue](https://github.com/RhetTbull/osxphotos/issues). Please consult the list of open bugs before deciding that you want to use this code on your Photos library. Notable issues include: - Face coordinates (mouth, left eye, right eye) may not be correct for images where the head is tilted. See [Issue #196](https://github.com/RhetTbull/osxphotos/issues/196). -- RAW images imported to Photos with an associated jpeg preview are not handled correctly by osxphotos. osxphotos query and export will operate on the jpeg preview instead of the RAW image as will `PhotoInfo.path`. If the user selects "Use RAW as original" in Photos, the RAW image will be exported or operated on but the jpeg will be ignored. See [Issue #101](https://github.com/RhetTbull/osxphotos/issues/101). Note: Beta version of fix for this bug is implemented in the current version of osxphotos. +- Raw images imported to Photos with an associated jpeg preview are not handled correctly by osxphotos. osxphotos query and export will operate on the jpeg preview instead of the raw image as will `PhotoInfo.path`. If the user selects "Use RAW as original" in Photos, the raw image will be exported or operated on but the jpeg will be ignored. See [Issue #101](https://github.com/RhetTbull/osxphotos/issues/101). Note: Beta version of fix for this bug is implemented in the current version of osxphotos. - The `--download-missing` option for `osxphotos export` does not work correctly with burst images. It will download the primary image but not the other burst images. See [Issue #75](https://github.com/RhetTbull/osxphotos/issues/75). ## Implementation Notes diff --git a/osxphotos/__main__.py b/osxphotos/__main__.py index a1f2233e..074b68f9 100644 --- a/osxphotos/__main__.py +++ b/osxphotos/__main__.py @@ -148,7 +148,7 @@ class ExportCommand(click.Command): formatter.write("\n") formatter.write_text( "Note: The number of files reported for export and the number actually exported " - + "may differ due to live photos, associated RAW images, and edited photos which are reported " + + "may differ due to live photos, associated raw images, and edited photos which are reported " + "in the total photos exported." ) formatter.write("\n") @@ -474,7 +474,7 @@ def query_options(f): o( "--has-raw", is_flag=True, - help="Search for photos with both a jpeg and RAW version", + help="Search for photos with both a jpeg and raw version", ), o( "--only-movies", @@ -1183,9 +1183,9 @@ def query( @click.option( "--skip-raw", is_flag=True, - help="Do not export associated RAW images of a RAW/jpeg pair. " - "Note: this does not skip RAW photos if the RAW photo does not have an associated jpeg image " - "(e.g. the RAW file was imported to Photos without a jpeg preview).", + help="Do not export associated raw images of a RAW+JPEG pair. " + "Note: this does not skip raw photos if the raw photo does not have an associated jpeg image " + "(e.g. the raw file was imported to Photos without a jpeg preview).", ) @click.option( "--person-keyword", @@ -1233,7 +1233,7 @@ def query( @click.option( "--convert-to-jpeg", is_flag=True, - help="Convert all non-jpeg images (e.g. RAW, HEIC, PNG, etc) " + help="Convert all non-jpeg images (e.g. raw, HEIC, PNG, etc) " "to JPEG upon export. Only works if your Mac has a GPU.", ) @click.option( @@ -1409,7 +1409,7 @@ def export( (e.g. search for photos matching all options). If no query options are provided, all photos will be exported. By default, all versions of all photos will be exported including edited - versions, live photo movies, burst photos, and associated RAW images. + versions, live photo movies, burst photos, and associated raw images. See --skip-edited, --skip-live, --skip-bursts, and --skip-raw options to modify this behavior. """ @@ -2058,7 +2058,7 @@ def _query( photos = [p for p in photos if not p.shared] if uti: - photos = [p for p in photos if uti in p.uti] + photos = [p for p in photos if uti in p.uti_original] if burst: photos = [p for p in photos if p.burst] @@ -2199,7 +2199,7 @@ def export_photo( directory: template used to determine output directory filename_template: template use to determine output file no_extended_attributes: boolean; if True, exports photo without preserving extended attributes - export_raw: boolean; if True exports RAW image associate with the photo + export_raw: boolean; if True exports raw image associate with the photo export_edited: boolean; if True exports edited version of photo if there is one skip_original_if_edited: boolean; if True does not export original if photo has been edited album_keyword: boolean; if True, exports album names as keywords in metadata diff --git a/osxphotos/photoinfo/photoinfo.py b/osxphotos/photoinfo/photoinfo.py index a29c6329..d11a3501 100644 --- a/osxphotos/photoinfo/photoinfo.py +++ b/osxphotos/photoinfo/photoinfo.py @@ -543,7 +543,10 @@ class PhotoInfo: """ Returns Uniform Type Identifier (UTI) for the original image for example: public.jpeg or com.apple.quicktime-movie """ - return self._info["UTI_original"] + if self._db._db_version <= _PHOTOS_4_VERSION and self._info["has_raw"]: + return self._info["raw_pair_info"]["UTI"] + else: + return self._info["UTI_original"] @property def uti_edited(self): @@ -745,12 +748,17 @@ class PhotoInfo: @property def has_raw(self): - """ returns True if photo has an associated RAW image, otherwise False """ + """ returns True if photo has an associated raw image (that is, it's a RAW+JPEG pair), otherwise False """ return self._info["has_raw"] + @property + def israw(self): + """ returns True if photo is a raw image. For images with an associated RAW+JPEG pair, see has_raw """ + return "raw-image" in self.uti_original + @property def raw_original(self): - """ returns True if associated RAW image and the RAW image is selected in Photos + """ returns True if associated raw image and the raw image is selected in Photos via "Use RAW as Original " otherwise returns False """ return self._info["raw_is_original"] diff --git a/osxphotos/photosdb/photosdb.py b/osxphotos/photosdb/photosdb.py index 30116091..448c45ce 100644 --- a/osxphotos/photosdb/photosdb.py +++ b/osxphotos/photosdb/photosdb.py @@ -1065,6 +1065,17 @@ class PhotosDB: self._dbphotos[uuid]["cloudAvailable"] = None self._dbphotos[uuid]["incloud"] = None + # associated RAW image info + self._dbphotos[uuid]["has_raw"] = True if row[25] == 7 else False + self._dbphotos[uuid]["UTI_raw"] = None + self._dbphotos[uuid]["raw_data_length"] = None + self._dbphotos[uuid]["raw_info"] = None + self._dbphotos[uuid]["resource_type"] = None # Photos 5 + self._dbphotos[uuid]["datastore_subtype"] = None # Photos 5 + self._dbphotos[uuid]["raw_master_uuid"] = row[29] + self._dbphotos[uuid]["non_raw_master_uuid"] = row[30] + self._dbphotos[uuid]["alt_master_uuid"] = row[31] + # original resource choice (e.g. RAW or jpeg) # In Photos 5+, original_resource_choice set from: # ZADDITIONALASSETATTRIBUTES.ZORIGINALRESOURCECHOICE @@ -1077,19 +1088,12 @@ class PhotosDB: # 64 = TIFF # 2048 = PNG # 32768 = HIEC - self._dbphotos[uuid]["original_resource_choice"] = 1 if row[40] == 16 else 0 - self._dbphotos[uuid]["raw_is_original"] = True if row[40] == 16 else False - - # associated RAW image info - self._dbphotos[uuid]["has_raw"] = True if row[25] == 7 else False - self._dbphotos[uuid]["UTI_raw"] = None - self._dbphotos[uuid]["raw_data_length"] = None - self._dbphotos[uuid]["raw_info"] = None - self._dbphotos[uuid]["resource_type"] = None # Photos 5 - self._dbphotos[uuid]["datastore_subtype"] = None # Photos 5 - self._dbphotos[uuid]["raw_master_uuid"] = row[29] - self._dbphotos[uuid]["non_raw_master_uuid"] = row[30] - self._dbphotos[uuid]["alt_master_uuid"] = row[31] + self._dbphotos[uuid]["original_resource_choice"] = ( + 1 if row[40] == 16 and self._dbphotos[uuid]["has_raw"] else 0 + ) + self._dbphotos[uuid]["raw_is_original"] = bool( + self._dbphotos[uuid]["original_resource_choice"] + ) # recently deleted items self._dbphotos[uuid]["intrash"] = True if row[32] == 1 else False @@ -2107,28 +2111,27 @@ class PhotosDB: WHERE ZDATASTORESUBTYPE = 1 OR ZDATASTORESUBTYPE = 3 """ ) + # Order of results: + # 0 {asset_table}.ZUUID, + # 1 ZINTERNALRESOURCE.ZLOCALAVAILABILITY, + # 2 ZINTERNALRESOURCE.ZREMOTEAVAILABILITY, + # 3 ZINTERNALRESOURCE.ZDATASTORESUBTYPE, + # 4 ZINTERNALRESOURCE.ZUNIFORMTYPEIDENTIFIER, + # 5 ZUNIFORMTYPEIDENTIFIER.ZIDENTIFIER + for row in c: uuid = row[0] if uuid in self._dbphotos: - # and self._dbphotos[uuid]["isMissing"] is None: - # logging.warning(f"uuid={uuid}, {row[1]}, {row[2]} {row[3]}") self._dbphotos[uuid]["localAvailability"] = row[1] self._dbphotos[uuid]["remoteAvailability"] = row[2] if row[3] == 1: self._dbphotos[uuid]["UTI_original"] = row[5] - # old = self._dbphotos[uuid]["isMissing"] - if row[1] != 1: self._dbphotos[uuid]["isMissing"] = 1 else: self._dbphotos[uuid]["isMissing"] = 0 - # if old is not None and old != self._dbphotos[uuid]["isMissing"]: - # logging.warning( - # f"{uuid} isMissing changed: {old} {self._dbphotos[uuid]['isMissing']}" - # ) - # get information on local/remote availability c.execute( f""" SELECT {asset_table}.ZUUID, @@ -2142,22 +2145,14 @@ class PhotosDB: for row in c: uuid = row[0] if uuid in self._dbphotos: - # logging.warning(f"uuid={uuid}, {row[1]}, {row[2]}") self._dbphotos[uuid]["localAvailability"] = row[1] self._dbphotos[uuid]["remoteAvailability"] = row[2] - # old = self._dbphotos[uuid]["isMissing"] - if row[1] != 1: self._dbphotos[uuid]["isMissing"] = 1 else: self._dbphotos[uuid]["isMissing"] = 0 - # if old is not None and old != self._dbphotos[uuid]["isMissing"]: - # logging.warning( - # f"{uuid} isMissing changed: {old} {self._dbphotos[uuid]['isMissing']}" - # ) - # get information about cloud sync state c.execute( f""" SELECT diff --git a/tests/Test-10.14.6.photoslibrary/database/photos.db b/tests/Test-10.14.6.photoslibrary/database/photos.db index ede28c3ed2564d9eef4199140c31af1b65dd6c27..d6d1d11dbc9c58ede6518aea20ab563c21f46fd2 100644 GIT binary patch delta 277 zcmX}hy-LGS9ER~G9&Mub`yc2++VtvDEh9E4$CT0jl%o1}% zns5l0m?sv9MIu9Fi5!t93Ph185oMx6R0)r$5lh4};on1?iEWB{?N&ZrQ%L(ebtgL1 zXA|-C@lbL)ewRDf+z2ywkai#Jvp#C^-kk@1-~Fi BVs!uj delta 277 zcmX}hKQ9Au9LMpy^P_jDs(;0OTXr_l2VfBrgQ1hqkNTe&cmPd%mwxsWk>4P{&88#< zxy8^2V5rbZJcDi(L?R(+;yt~SSG7^DHp-Eo&RdrKVf~j|JDXpy^8pB(aEKm45xqno zksw^cBl?K}VvtA@Lqv)gCep+RF-nXPT~P z7(aYT$#%>^Jo_!Hf3qLoyvmZT!o1njZ;`}0x|WIx7R|11|48hZo%pdX8+I*Q%U#Xj E7ua59!2kdN diff --git a/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist b/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist index 02189263..ed56bc8a 100644 --- a/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist +++ b/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotoAnalysisServicePreferences.plist @@ -5,6 +5,6 @@ PhotoAnalysisGraphLastBackgroundGraphRebuildJobDate 2020-10-09T16:14:42Z PhotoAnalysisGraphLastBackgroundMemoryGenerationJobDate - 2020-10-09T19:48:34Z + 2020-10-10T05:21:03Z diff --git a/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotosGraph/photosgraph.graphdb b/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotosGraph/photosgraph.graphdb index 8b42ee7aecf3f23d8cbd68dbc63fe1a3da4200c8..ecb7f869a4bdc169099121709e8e2c0e88a6675b 100644 GIT binary patch delta 698 zcmY+AO>5Lp6o$`9NYZ-{Vx&8}C|J}ELfg%BKsQCJcj5<%AXsee2-!%{g;4D{#R=MC z?WU=VK{sx6*}@RSjVpE0PCw=kNGYY@KbW?T119Gnl1(1&dvot|&s*KKs=Lr`g9bKeV1tHMy!>#;ai?gI zdXpIz=`xMe*?M%N>uqn@deMN6V`>)@ybavK z1nlwd8nl^nu%90McX$I1INp@U55kINA`bPW-}qyB#&W-^52~y_s|B^Bo~YIMT=ly0 z`RYK#7|HyBToH&UNx9-c1j&*c2&92P8VHgGzAyY3#ZLUXd9QTbUUlZ|U-oi*}C9`vjnKnKtyE((9i=TUO1D%dF}$#x$u_l7__8vLw+nv( D5#Ete delta 698 zcmY+A%}W(g7>D0?hMDoa1k*&jTnI_!B~Ull3$>B_ICuJ>2!bR(t}vSrt;~fN!#k9c zxp8c9$&H&@`Y|4_ZCX{UHvI!3B7!@;sdv?;ne!4coB45`Gv_?-`BnC;%AU1!39YF4 z7FzMhr~B>*_0T(dOt)!T3CEr{gXr zlMhk{_0ctIp|Lo+*Yv)nY|Yl-)G?EHY{Qw(EvToH4Z|;b1P%abs^CQDJnYiUe;k8u zVF>p1?hYL3oP&e(;J?G0(9n2;hwp$j%S7yKPQU77dD?Q9#H9El7DQRBi<($f*DH60 zudfbRj1j*-;41!hOn7cBS;hWNL_rm8Hwjs~?pVzwZVc{=V Ck&br& diff --git a/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotosGraph/photosgraph.graphdb-shm b/tests/Test-10.14.6.photoslibrary/private/com.apple.photoanalysisd/GraphService/PhotosGraph/photosgraph.graphdb-shm index 87d3e641116d5d023833d8a61300f93a8c83fe42..fe9ac2845eca6fe6da8a63cd096d9cf9e24ece10 100644 GIT binary patch delta 89 zcmZo@U}|V!;+1%$%K!t66BkO0TChv7nNGgVi7uf4lYpuIj|Mg_jCI<$LE9Dp%F!O} delta 215 zcmZo@U}|V!s+V}A%K!t63=9J7K#n*N%ZF``U!b`=cYjdl?U%}pWwr*V`&1K})JRni zG7khm^nWA(7Gz*h0!x56K+73;ftU@5L5ewmm=lOaHa5mOF*0s!{L073ys_~w6C(?l ZWCfFK8ymlKGcs*#{3yc6zOhlA1pvlYH|+ob diff --git a/tests/Test-10.14.6.photoslibrary/resources/recovery/Info.plist b/tests/Test-10.14.6.photoslibrary/resources/recovery/Info.plist index 1d2de6d3..facca9d6 100644 --- a/tests/Test-10.14.6.photoslibrary/resources/recovery/Info.plist +++ b/tests/Test-10.14.6.photoslibrary/resources/recovery/Info.plist @@ -24,7 +24,7 @@ SnapshotCompletedDate 2019-07-27T13:16:43Z SnapshotLastValidated - 2020-10-09T16:16:27Z + 2020-10-10T05:22:36Z SnapshotTables diff --git a/tests/test_bigsur_10_16_0.py b/tests/test_bigsur_10_16_0.py index 8a84d6a8..58aaee93 100644 --- a/tests/test_bigsur_10_16_0.py +++ b/tests/test_bigsur_10_16_0.py @@ -1,5 +1,7 @@ import pytest +from collections import namedtuple + from osxphotos._constants import _UNKNOWN_PERSON @@ -95,13 +97,70 @@ UTI_DICT = { "8846E3E6-8AC8-4857-8448-E3D025784410": "public.tiff", "7783E8E6-9CAC-40F3-BE22-81FB7051C266": "public.heic", "1EB2B765-0765-43BA-A90C-0D0580E6172C": "public.jpeg", + "4D521201-92AC-43E5-8F7C-59BC41C37A96": "public.jpeg", } - UTI_ORIGINAL_DICT = { "8846E3E6-8AC8-4857-8448-E3D025784410": "public.tiff", "7783E8E6-9CAC-40F3-BE22-81FB7051C266": "public.heic", "1EB2B765-0765-43BA-A90C-0D0580E6172C": "public.jpeg", + "4D521201-92AC-43E5-8F7C-59BC41C37A96": "public.jpeg", +} + +RawInfo = namedtuple( + "RawInfo", + [ + "comment", + "original_filename", + "has_raw", + "israw", + "raw_original", + "uti", + "uti_original", + "uti_raw", + ], +) +RAW_DICT = { + "D05A5FE3-15FB-49A1-A15D-AB3DA6F8B068": RawInfo( + "raw image, no jpeg pair", + "DSC03584.dng", + False, + True, + False, + "com.adobe.raw-image", + "com.adobe.raw-image", + None, + ), + "A92D9C26-3A50-4197-9388-CB5F7DB9FA91": RawInfo( + "raw+jpeg, jpeg original", + "IMG_1994.JPG", + True, + False, + False, + "public.jpeg", + "public.jpeg", + "com.canon.cr2-raw-image", + ), + "4D521201-92AC-43E5-8F7C-59BC41C37A96": RawInfo( + "raw+jpeg, raw original", + "IMG_1997.JPG", + True, + False, + True, + "public.jpeg", + "public.jpeg", + "com.canon.cr2-raw-image", + ), + "E9BC5C36-7CD1-40A1-A72B-8B8FAC227D51": RawInfo( + "jpeg, no raw", + "wedding.jpg", + False, + False, + False, + "public.jpeg", + "public.jpeg", + None, + ), } # HEIC image that's been edited in Big Sur, resulting edit is .HEIC @@ -1115,3 +1174,19 @@ def test_uti(): photo = photosdb.get_photo(uuid) assert photo.uti == uti assert photo.uti_original == UTI_ORIGINAL_DICT[uuid] + + +def test_raw(): + """ Test various raw properties """ + import osxphotos + + photosdb = osxphotos.PhotosDB(dbfile=PHOTOS_DB) + + for uuid, rawinfo in RAW_DICT.items(): + photo = photosdb.get_photo(uuid) + assert photo.original_filename == rawinfo.original_filename + assert photo.has_raw == rawinfo.has_raw + assert photo.israw == rawinfo.israw + assert photo.uti == rawinfo.uti + assert photo.uti_original == rawinfo.uti_original + assert photo.uti_raw == rawinfo.uti_raw diff --git a/tests/test_mojave_10_14_6.py b/tests/test_mojave_10_14_6.py index 2fe39af7..967abad8 100644 --- a/tests/test_mojave_10_14_6.py +++ b/tests/test_mojave_10_14_6.py @@ -78,7 +78,7 @@ UUID_UTI_DICT = { "oTiMG6OfSP6d%nUTEOfvMg": [ "public.jpeg", "com.canon.cr2-raw-image", - "com.canon.cr2-raw-image", + "public.jpeg", None, ], "GdJJPQX0RP63mcdKFj%sfQ": ["public.jpeg", None, "public.heic", "public.jpeg"],