Initial FaceInfo support for Issue #21

This commit is contained in:
Rhet Turnbull
2020-07-27 06:20:04 -07:00
parent 9fc4f76219
commit 6f29cda99f
377 changed files with 3712 additions and 18 deletions

View File

@@ -1,3 +1,3 @@
""" version info """
__version__ = "0.30.13"
__version__ = "0.31.0"

View File

@@ -1,7 +1,8 @@
""" PhotoInfo methods to expose info about person in the Photos library """
""" PhotoInfo and FaceInfo classes to expose info about persons and faces in the Photos library """
import json
import logging
import math
class PersonInfo:
@@ -49,6 +50,22 @@ class PersonInfo:
""" Returns list of PhotoInfo objects associated with this person """
return self._db.photos_by_uuid(self._db._dbfaces_pk[self._pk])
@property
def face_info(self):
""" Returns a list of FaceInfo objects associated with this person sorted by quality score
Highest quality face is result[0] and lowest quality face is result[n]
"""
try:
faces = self._db._db_faceinfo_person[self._pk]
return sorted(
[FaceInfo(db=self._db, pk=face) for face in faces],
key=lambda face: face.quality,
reverse=True,
)
except KeyError:
# no faces
return []
def json(self):
""" Returns JSON representation of class instance """
keyphoto = self.keyphoto.uuid if self.keyphoto is not None else None
@@ -75,3 +92,317 @@ class PersonInfo:
def __ne__(self, other):
return not self.__eq__(other)
class FaceInfo:
""" Info about a face in the Photos library
"""
def __init__(self, db=None, pk=None):
""" Creates a new FaceInfo instance
Arguments:
db: instance of PhotosDB object
pk: primary key value of face to init the object with
Returns:
FaceInfo instance
"""
self._db = db
self._pk = pk
face = self._db._db_faceinfo_pk[pk]
self._info = face
self.uuid = face["uuid"]
self.name = face["fullname"]
self.asset_uuid = face["asset_uuid"]
self._person_pk = face["person"]
self.center_x = face["centerx"]
self.center_y = face["centery"]
self.mouth_x = face["mouthx"]
self.mouth_y = face["mouthy"]
self.left_eye_x = face["lefteyex"]
self.left_eye_y = face["lefteyey"]
self.right_eye_x = face["righteyex"]
self.right_eye_y = face["righteyey"]
self.size = face["size"]
self.quality = face["quality"]
self.source_width = face["sourcewidth"]
self.source_height = face["sourceheight"]
self.has_smile = face["has_smile"]
self.left_eye_closed = face["left_eye_closed"]
self.right_eye_closed = face["right_eye_closed"]
self.manual = face["manual"]
self.face_type = face["facetype"]
self.age_type = face["agetype"]
self.bald_type = face["baldtype"]
self.eye_makeup_type = face["eyemakeuptype"]
self.eye_state = face["eyestate"]
self.facial_hair_type = face["facialhairtype"]
self.gender_type = face["gendertype"]
self.glasses_type = face["glassestype"]
self.hair_color_type = face["haircolortype"]
self.intrash = face["intrash"]
self.lip_makeup_type = face["lipmakeuptype"]
self.smile_type = face["smiletype"]
@property
def center(self):
""" Coordinates, in PIL format, for center of face
Returns:
tuple of coordinates in form (x, y)
"""
return self._make_point((self.center_x, self.center_y))
@property
def size_pixels(self):
""" Size of face in pixels (centered around center_x, center_y)
Returns:
size, in int pixels, of a circle drawn around the center of the face
"""
photo = self.photo
size_reference = photo.width if photo.width > photo.height else photo.height
return self.size * size_reference
@property
def mouth(self):
""" Coordinates, in PIL format, for mouth position
Returns:
tuple of coordinates in form (x, y)
"""
return self._make_point_with_rotation((self.mouth_x, self.mouth_y))
@property
def left_eye(self):
""" Coordinates, in PIL format, for left eye position
Returns:
tuple of coordinates in form (x, y)
"""
return self._make_point_with_rotation((self.left_eye_x, self.left_eye_y))
@property
def right_eye(self):
""" Coordinates, in PIL format, for right eye position
Returns:
tuple of coordinates in form (x, y)
"""
return self._make_point_with_rotation((self.right_eye_x, self.right_eye_y))
@property
def person_info(self):
""" PersonInfo instance for person associated with this face """
try:
return self._person
except AttributeError:
self._person = PersonInfo(db=self._db, pk=self._person_pk)
return self._person
@property
def photo(self):
""" PhotoInfo instance associated with this face """
try:
return self._photo
except AttributeError:
self._photo = self._db.get_photo(self.asset_uuid)
if self._photo is None:
logging.warning(f"Could not get photo for uuid: {self.asset_uuid}")
return self._photo
def face_rect(self):
""" Get face rectangle coordinates for current version of the associated image
If image has been edited, rectangle applies to edited version, otherwise original version
Coordinates in format and reference frame used by PIL
Returns:
list [(x0, x1), (y0, y1)] of coordinates in reference frame used by PIL
"""
photo = self.photo
size_reference = photo.width if photo.width > photo.height else photo.height
radius = (self.size / 2) * size_reference
x, y = self._make_point((self.center_x, self.center_y))
x0, y0 = x - radius, y - radius
x1, y1 = x + radius, y + radius
return [(x0, y0), (x1, y1)]
def roll_pitch_yaw(self):
""" Roll, pitch, yaw of face in radians as tuple """
info = self._info
roll = 0 if info["roll"] is None else info["roll"]
pitch = 0 if info["pitch"] is None else info["pitch"]
yaw = 0 if info["yaw"] is None else info["yaw"]
return (roll, pitch, yaw)
@property
def roll(self):
""" Return roll angle in radians of the face region """
roll, _, _ = self.roll_pitch_yaw()
return roll
@property
def pitch(self):
""" Return pitch angle in radians of the face region """
_, pitch, _ = self.roll_pitch_yaw()
return pitch
@property
def yaw(self):
""" Return yaw angle in radians of the face region """
_, _, yaw = self.roll_pitch_yaw()
return yaw
def _make_point(self, xy):
""" Translate an (x, y) tuple based on image orientation
and convert to image coordinates
Arguments:
xy: tuple of (x, y) coordinates for point to translate
in format used by Photos (percent of height/width)
Returns:
(x, y) tuple of translated coordinates in pixels in PIL format/reference frame
"""
# Reference: https://github.com/neilpa/phace/blob/7594776480505d0c389688a42099c94ac5d34f3f/cmd/phace/draw.go#L79-L94
orientation = self.photo.orientation
x, y = xy
dx = self.photo.width
dy = self.photo.height
if orientation in [1, 2]:
y = 1.0 - y
elif orientation in [3, 4]:
x = 1.0 - x
elif orientation in [5, 6]:
x, y = 1.0 - y, 1.0 - x
dx, dy = dy, dx
elif orientation in [7, 8]:
x, y = y, x
dx, dy = dy, dx
else:
logging.warning(f"Unhandled orientation: {orientation}")
return (int(x * dx), int(y * dy))
def _make_point_with_rotation(self, xy):
""" Translate an (x, y) tuple based on image orientation and rotation
and convert to image coordinates
Arguments:
xy: tuple of (x, y) coordinates for point to translate
in format used by Photos (percent of height/width)
Returns:
(x, y) tuple of translated coordinates in pixels in PIL format/reference frame
"""
# convert to image coordinates
x, y = self._make_point(xy)
# rotate about center
xmid, ymid = self.center
roll, _, _ = self.roll_pitch_yaw()
xr, yr = rotate_image_point(x, y, xmid, ymid, roll)
return (int(xr), int(yr))
def asdict(self):
""" Returns dict representation of class instance """
roll, pitch, yaw = self.roll_pitch_yaw()
return {
"_pk": self._pk,
"uuid": self.uuid,
"name": self.name,
"asset_uuid": self.asset_uuid,
"_person_pk": self._person_pk,
"center_x": self.center_x,
"center_y": self.center_y,
"center": self.center,
"mouth_x": self.mouth_x,
"mouth_y": self.mouth_y,
"mouth": self.mouth,
"left_eye_x": self.left_eye_x,
"left_eye_y": self.left_eye_y,
"left_eye": self.left_eye,
"right_eye_x": self.right_eye_x,
"right_eye_y": self.right_eye_y,
"right_eye": self.right_eye,
"size": self.size,
"face_rect": self.face_rect(),
"roll": roll,
"pitch": pitch,
"yaw": yaw,
"quality": self.quality,
"source_width": self.source_width,
"source_height": self.source_height,
"has_smile": self.has_smile,
"left_eye_closed": self.left_eye_closed,
"right_eye_closed": self.right_eye_closed,
"manual": self.manual,
"face_type": self.face_type,
"age_type": self.age_type,
"bald_type": self.bald_type,
"eye_makeup_type": self.eye_makeup_type,
"eye_state": self.eye_state,
"facial_hair_type": self.facial_hair_type,
"gender_type": self.gender_type,
"glasses_type": self.glasses_type,
"hair_color_type": self.hair_color_type,
"intrash": self.intrash,
"lip_makeup_type": self.lip_makeup_type,
"smile_type": self.smile_type,
}
def json(self):
""" Return JSON representation of FaceInfo instance """
return json.dumps(self.asdict())
def __str__(self):
return f"FaceInfo(uuid={self.uuid}, center_x={self.center_x}, center_y = {self.center_y}, size={self.size}, person={self.name}, asset_uuid={self.asset_uuid})"
def __repr__(self):
return f"FaceInfo(db={self._db}, pk={self._pk})"
def __eq__(self, other):
if not isinstance(other, type(self)):
return False
return all(
getattr(self, field) == getattr(other, field) for field in ["_db", "_pk"]
)
def __ne__(self, other):
return not self.__eq__(other)
def rotate_image_point(x, y, xmid, ymid, angle):
""" rotate image point about xm, ym by angle in radians
Arguments:
x: x coordinate of point to rotate
y: y coordinate of point to rotate
xmid: x coordinate of center point to rotate about
ymid: y coordinate of center point to rotate about
angle: angle in radians about which to coordinate,
counter-clockwise is positive
Returns:
tuple of rotated points (xr, yr)
"""
# translate point relative to the mid point
x = x - xmid
y = y - ymid
# rotate by angle and translate back
# the photo coordinate system is downwards y is positive so
# need to adjust the rotation accordingly
cos_angle = math.cos(angle)
sin_angle = math.sin(angle)
xr = x * cos_angle + y * sin_angle + xmid
yr = -x * sin_angle + y * cos_angle + ymid
return (xr, yr)

View File

@@ -29,7 +29,7 @@ from .._constants import (
_PHOTOS_5_SHARED_PHOTO_PATH,
)
from ..albuminfo import AlbumInfo
from ..personinfo import PersonInfo
from ..personinfo import FaceInfo, PersonInfo
from ..phototemplate import PhotoTemplate
from ..placeinfo import PlaceInfo4, PlaceInfo5
from ..utils import _debug, _get_resource_loc, findfiles, get_preferred_uti_extension
@@ -353,6 +353,20 @@ class PhotoInfo:
]
return self._personinfo
@property
def face_info(self):
""" list of FaceInfo objects for faces in picture """
try:
return self._faceinfo
except AttributeError:
try:
faces = self._db._db_faceinfo_uuid[self._uuid]
self._faceinfo = [FaceInfo(db=self._db, pk=pk) for pk in faces]
except KeyError:
# no faces
self._faceinfo = []
return self._faceinfo
@property
def albums(self):
""" list of albums picture is contained in """

View File

@@ -54,3 +54,5 @@ def _process_exifinfo_5(photosdb):
if uuid in photosdb._db_exifinfo_uuid:
logging.warning(f"duplicate exifinfo record found for uuid {uuid}")
photosdb._db_exifinfo_uuid[uuid] = record
conn.close()

View File

@@ -0,0 +1,328 @@
""" Methods for PhotosDB to add Photos face info
"""
import logging
from .._constants import _PHOTOS_4_VERSION
from ..utils import _open_sql_file
"""
This module should be imported in the class defintion of PhotosDB in photosdb.py
Do not import this module directly
This module adds the following method to PhotosDB:
_process_faceinfo: process photo face info
The following data structures are added to PhotosDB
self._db_faceinfo_pk: {pk: {faceinfo}}
self._db_faceinfo_uuid: {photo uuid: [face pk]}
self._db_faceinfo_person: {person_pk: [face_pk]}
"""
def _process_faceinfo(self):
""" Process face information
"""
self._db_faceinfo_pk = {}
self._db_faceinfo_uuid = {}
self._db_faceinfo_person = {}
if self._db_version <= _PHOTOS_4_VERSION:
_process_faceinfo_4(self)
else:
_process_faceinfo_5(self)
def _process_faceinfo_4(photosdb):
""" Process face information for Photos 4 databases
Args:
photosdb: an OSXPhotosDB instance
"""
db = photosdb._tmp_db
(conn, cursor) = _open_sql_file(db)
result = cursor.execute(
"""
SELECT
RKFace.modelId,
RKVersion.uuid,
RKFace.uuid,
RKPerson.name,
RKFace.isInTrash,
RKFace.personId,
RKFace.imageModelId,
RKFace.sourceWidth,
RKFace.sourceHeight,
RKFace.centerX,
RKFace.centerY,
RKFace.size,
RKFace.leftEyeX,
RKFace.leftEyeY,
RKFace.rightEyeX,
RKFace.rightEyeY,
RKFace.mouthX,
RKFace.mouthY,
RKFace.hidden,
RKFace.manual,
RKFace.hasSmile,
RKFace.isLeftEyeClosed,
RKFace.isRightEyeClosed,
RKFace.poseRoll,
RKFace.poseYaw,
RKFace.posePitch,
RKFace.faceType,
RKFace.qualityMeasure
FROM
RKFace
JOIN RKPerson on RKPerson.modelId = RKFace.personId
JOIN RKVersion on RKVersion.modelId = RKFace.imageModelId
"""
)
# 0 RKFace.modelId,
# 1 RKVersion.uuid,
# 2 RKFace.uuid,
# 3 RKPerson.name,
# 4 RKFace.isInTrash,
# 5 RKFace.personId,
# 6 RKFace.imageModelId,
# 7 RKFace.sourceWidth,
# 8 RKFace.sourceHeight,
# 9 RKFace.centerX,
# 10 RKFace.centerY,
# 11 RKFace.size,
# 12 RKFace.leftEyeX,
# 13 RKFace.leftEyeY,
# 14 RKFace.rightEyeX,
# 15 RKFace.rightEyeY,
# 16 RKFace.mouthX,
# 17 RKFace.mouthY,
# 18 RKFace.hidden,
# 19 RKFace.manual,
# 20 RKFace.hasSmile,
# 21 RKFace.isLeftEyeClosed,
# 22 RKFace.isRightEyeClosed,
# 23 RKFace.poseRoll,
# 24 RKFace.poseYaw,
# 25 RKFace.posePitch,
# 26 RKFace.faceType,
# 27 RKFace.qualityMeasure
for row in result:
modelid = row[0]
asset_uuid = row[1]
person_id = row[5]
face = {}
face["pk"] = modelid
face["asset_uuid"] = asset_uuid
face["uuid"] = row[2]
face["person"] = person_id
face["fullname"] = row[3]
face["sourcewidth"] = row[7]
face["sourceheight"] = row[8]
face["centerx"] = row[9]
face["centery"] = row[10]
face["size"] = row[11]
face["lefteyex"] = row[12]
face["lefteyey"] = row[13]
face["righteyex"] = row[14]
face["righteyey"] = row[15]
face["mouthx"] = row[16]
face["mouthy"] = row[17]
face["hidden"] = row[18]
face["manual"] = row[19]
face["has_smile"] = row[20]
face["left_eye_closed"] = row[21]
face["right_eye_closed"] = row[22]
face["roll"] = row[23]
face["yaw"] = row[24]
face["pitch"] = row[25]
face["facetype"] = row[26]
face["quality"] = row[27]
# Photos 5 only
face["agetype"] = None
face["baldtype"] = None
face["eyemakeuptype"] = None
face["eyestate"] = None
face["facialhairtype"] = None
face["gendertype"] = None
face["glassestype"] = None
face["haircolortype"] = None
face["intrash"] = None
face["lipmakeuptype"] = None
face["smiletype"] = None
photosdb._db_faceinfo_pk[modelid] = face
try:
photosdb._db_faceinfo_uuid[asset_uuid].append(modelid)
except KeyError:
photosdb._db_faceinfo_uuid[asset_uuid] = [modelid]
try:
photosdb._db_faceinfo_person[person_id].append(modelid)
except KeyError:
photosdb._db_faceinfo_person[person_id] = [modelid]
conn.close()
def _process_faceinfo_5(photosdb):
""" Process face information for Photos 5 databases
Args:
photosdb: an OSXPhotosDB instance
"""
db = photosdb._tmp_db
(conn, cursor) = _open_sql_file(db)
result = cursor.execute(
"""
SELECT
ZDETECTEDFACE.Z_PK,
ZGENERICASSET.ZUUID,
ZDETECTEDFACE.ZUUID,
ZDETECTEDFACE.ZPERSON,
ZPERSON.ZFULLNAME,
ZDETECTEDFACE.ZAGETYPE,
ZDETECTEDFACE.ZBALDTYPE,
ZDETECTEDFACE.ZEYEMAKEUPTYPE,
ZDETECTEDFACE.ZEYESSTATE,
ZDETECTEDFACE.ZFACIALHAIRTYPE,
ZDETECTEDFACE.ZGENDERTYPE,
ZDETECTEDFACE.ZGLASSESTYPE,
ZDETECTEDFACE.ZHAIRCOLORTYPE,
ZDETECTEDFACE.ZHASSMILE,
ZDETECTEDFACE.ZHIDDEN,
ZDETECTEDFACE.ZISINTRASH,
ZDETECTEDFACE.ZISLEFTEYECLOSED,
ZDETECTEDFACE.ZISRIGHTEYECLOSED,
ZDETECTEDFACE.ZLIPMAKEUPTYPE,
ZDETECTEDFACE.ZMANUAL,
ZDETECTEDFACE.ZQUALITYMEASURE,
ZDETECTEDFACE.ZSMILETYPE,
ZDETECTEDFACE.ZSOURCEHEIGHT,
ZDETECTEDFACE.ZSOURCEWIDTH,
ZDETECTEDFACE.ZBLURSCORE,
ZDETECTEDFACE.ZCENTERX,
ZDETECTEDFACE.ZCENTERY,
ZDETECTEDFACE.ZLEFTEYEX,
ZDETECTEDFACE.ZLEFTEYEY,
ZDETECTEDFACE.ZMOUTHX,
ZDETECTEDFACE.ZMOUTHY,
ZDETECTEDFACE.ZPOSEYAW,
ZDETECTEDFACE.ZQUALITY,
ZDETECTEDFACE.ZRIGHTEYEX,
ZDETECTEDFACE.ZRIGHTEYEY,
ZDETECTEDFACE.ZROLL,
ZDETECTEDFACE.ZSIZE,
ZDETECTEDFACE.ZYAW,
ZDETECTEDFACE.ZMASTERIDENTIFIER
FROM ZDETECTEDFACE
JOIN ZGENERICASSET ON ZGENERICASSET.Z_PK = ZDETECTEDFACE.ZASSET
JOIN ZPERSON ON ZPERSON.Z_PK = ZDETECTEDFACE.ZPERSON;
"""
)
# 0 ZDETECTEDFACE.Z_PK
# 1 ZGENERICASSET.ZUUID,
# 2 ZDETECTEDFACE.ZUUID,
# 3 ZDETECTEDFACE.ZPERSON,
# 4 ZPERSON.ZFULLNAME,
# 5 ZDETECTEDFACE.ZAGETYPE,
# 6 ZDETECTEDFACE.ZBALDTYPE,
# 7 ZDETECTEDFACE.ZEYEMAKEUPTYPE,
# 8 ZDETECTEDFACE.ZEYESSTATE,
# 9 ZDETECTEDFACE.ZFACIALHAIRTYPE,
# 10 ZDETECTEDFACE.ZGENDERTYPE,
# 11 ZDETECTEDFACE.ZGLASSESTYPE,
# 12 ZDETECTEDFACE.ZHAIRCOLORTYPE,
# 13 ZDETECTEDFACE.ZHASSMILE,
# 14 ZDETECTEDFACE.ZHIDDEN,
# 15 ZDETECTEDFACE.ZISINTRASH,
# 16 ZDETECTEDFACE.ZISLEFTEYECLOSED,
# 17 ZDETECTEDFACE.ZISRIGHTEYECLOSED,
# 18 ZDETECTEDFACE.ZLIPMAKEUPTYPE,
# 19 ZDETECTEDFACE.ZMANUAL,
# 20 ZDETECTEDFACE.ZQUALITYMEASURE,
# 21 ZDETECTEDFACE.ZSMILETYPE,
# 22 ZDETECTEDFACE.ZSOURCEHEIGHT,
# 23 ZDETECTEDFACE.ZSOURCEWIDTH,
# 24 ZDETECTEDFACE.ZBLURSCORE,
# 25 ZDETECTEDFACE.ZCENTERX,
# 26 ZDETECTEDFACE.ZCENTERY,
# 27 ZDETECTEDFACE.ZLEFTEYEX,
# 28 ZDETECTEDFACE.ZLEFTEYEY,
# 29 ZDETECTEDFACE.ZMOUTHX,
# 30 ZDETECTEDFACE.ZMOUTHY,
# 31 ZDETECTEDFACE.ZPOSEYAW,
# 32 ZDETECTEDFACE.ZQUALITY,
# 33 ZDETECTEDFACE.ZRIGHTEYEX,
# 34 ZDETECTEDFACE.ZRIGHTEYEY,
# 35 ZDETECTEDFACE.ZROLL,
# 36 ZDETECTEDFACE.ZSIZE,
# 37 ZDETECTEDFACE.ZYAW,
# 38 ZDETECTEDFACE.ZMASTERIDENTIFIER
for row in result:
pk = row[0]
asset_uuid = row[1]
person_pk = row[3]
face = {}
face["pk"] = pk
face["asset_uuid"] = asset_uuid
face["uuid"] = row[2]
face["person"] = person_pk
face["fullname"] = row[4]
face["agetype"] = row[5]
face["baldtype"] = row[6]
face["eyemakeuptype"] = row[7]
face["eyestate"] = row[8]
face["facialhairtype"] = row[9]
face["gendertype"] = row[10]
face["glassestype"] = row[11]
face["haircolortype"] = row[12]
face["has_smile"] = row[13]
face["hidden"] = row[14]
face["intrash"] = row[15]
face["left_eye_closed"] = row[16]
face["right_eye_closed"] = row[17]
face["lipmakeuptype"] = row[18]
face["manual"] = row[19]
face["smiletype"] = row[21]
face["sourceheight"] = row[22]
face["sourcewidth"] = row[23]
face["facetype"] = None # Photos 4 only
face["centerx"] = row[25]
face["centery"] = row[26]
face["lefteyex"] = row[27]
face["lefteyey"] = row[28]
face["mouthx"] = row[29]
face["mouthy"] = row[30]
face["quality"] = row[32]
face["righteyex"] = row[33]
face["righteyey"] = row[34]
face["roll"] = row[35]
face["size"] = row[36]
face["yaw"] = row[37]
face["pitch"] = 0.0 # not defined in Photos 5
photosdb._db_faceinfo_pk[pk] = face
try:
photosdb._db_faceinfo_uuid[asset_uuid].append(pk)
except KeyError:
photosdb._db_faceinfo_uuid[asset_uuid] = [pk]
try:
photosdb._db_faceinfo_person[person_pk].append(pk)
except KeyError:
photosdb._db_faceinfo_person[person_pk] = [pk]
conn.close()

View File

@@ -143,3 +143,5 @@ def _process_scoreinfo_5(photosdb):
scores["well_framed_subject"] = row[26]
scores["well_timed_shot"] = row[27]
photosdb._db_scoreinfo_uuid[uuid] = scores
conn.close()

View File

@@ -147,7 +147,8 @@ def _process_searchinfo(self):
"_db_searchinfo_labels_normalized: \n"
+ pformat(self._db_searchinfo_labels_normalized)
)
conn.close()
@property
def labels(self):

View File

@@ -55,6 +55,7 @@ class PhotosDB:
# import additional methods
from ._photosdb_process_exif import _process_exifinfo
from ._photosdb_process_faceinfo import _process_faceinfo
from ._photosdb_process_searchinfo import (
_process_searchinfo,
labels,
@@ -1326,6 +1327,9 @@ class PhotosDB:
# done with the database connection
conn.close()
# process faces
self._process_faceinfo()
# add faces and keywords to photo data
for uuid in self._dbphotos:
# keywords
@@ -2129,6 +2133,9 @@ class PhotosDB:
# close connection and remove temporary files
conn.close()
# process face info
self._process_faceinfo()
# process search info
self._process_searchinfo()