forked/osxphotos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

performance improvements for update and export_db

Browse Source
This commit is contained in:
Rhet Turnbull
2020-05-27 20:52:05 -07:00
parent 9b11cbf32b
commit 42b89d34f3
4 changed files with 117 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
osxphotos/_export_db.py
View File

@@ -19,6 +19,7 @@ OSXPHOTOS_EXPORTDB_VERSION = "1.0"
class ExportDB_ABC(ABC):
""" abstract base class for ExportDB """
@abstractmethod
def get_uuid_for_file(self, filename):
pass
@@ -306,17 +307,39 @@ class ExportDB(ExportDB_ABC):
def set_data(self, filename, uuid, orig_stat, exif_stat, info_json, exif_json):
""" sets all the data for file and uuid at once
calls set_uuid_for_file
set_info_for_uuid
set_stat_orig_for_file
set_stat_exif_for_file
set_exifdata_for_file
"""
self.set_uuid_for_file(filename, uuid)
self.set_info_for_uuid(uuid, info_json)
self.set_stat_orig_for_file(filename, orig_stat)
self.set_stat_exif_for_file(filename, exif_stat)
self.set_exifdata_for_file(filename, exif_json)
filename = str(pathlib.Path(filename).relative_to(self._path))
filename_normalized = filename.lower()
conn = self._conn
try:
c = conn.cursor()
c.execute(
f"INSERT OR REPLACE INTO files(filepath, filepath_normalized, uuid) VALUES (?, ?, ?);",
(filename, filename_normalized, uuid),
)
c.execute(
"UPDATE files "
+ "SET orig_mode = ?, orig_size = ?, orig_mtime = ? "
+ "WHERE filepath_normalized = ?;",
(*orig_stat, filename_normalized),
)
c.execute(
"UPDATE files "
+ "SET exif_mode = ?, exif_size = ?, exif_mtime = ? "
+ "WHERE filepath_normalized = ?;",
(*exif_stat, filename_normalized),
)
c.execute(
"INSERT OR REPLACE INTO info(uuid, json_info) VALUES (?, ?);",
(uuid, info_json),
)
c.execute(
"INSERT OR REPLACE INTO exifdata(filepath_normalized, json_exifdata) VALUES (?, ?);",
(filename_normalized, exif_json),
)
conn.commit()
except Error as e:
logging.warning(e)
def close(self):
""" close the database connection """
@@ -475,7 +498,7 @@ class ExportDBInMemory(ExportDB):
except Error as e:
logging.warning(e)
raise e
tempfile = StringIO()
for line in conn.iterdump():
tempfile.write("%s\n" % line)

104
osxphotos/photoinfo/_photoinfo_export.py
View File

@@ -452,11 +452,14 @@ def export2(
f"Found matching file with blank uuid: {self.uuid}, {dest}"
)
dest_uuid = self.uuid
export_db.set_uuid_for_file(dest, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest, fileutil.file_sig(dest))
export_db.set_stat_exif_for_file(dest, (None, None, None))
export_db.set_exifdata_for_file(dest, None)
export_db.set_data(
dest,
self.uuid,
fileutil.file_sig(dest),
(None, None, None),
self.json(),
None,
)
if dest_uuid != self.uuid:
# not the right file, find the right one
logging.debug(
@@ -482,11 +485,14 @@ def export2(
)
dest = pathlib.Path(file_)
found_match = True
export_db.set_uuid_for_file(file_, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest, fileutil.file_sig(dest))
export_db.set_stat_exif_for_file(dest, (None, None, None))
export_db.set_exifdata_for_file(dest, None)
export_db.set_data(
dest,
self.uuid,
fileutil.file_sig(dest),
(None, None, None),
self.json(),
None,
)
break
if not found_match:
@@ -780,11 +786,14 @@ def _export_photo(
fileutil.unlink(dest)
logging.debug(f"Not update: export_as_hardlink linking file {src} {dest}")
fileutil.hardlink(src, dest)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
exported_files.append(dest_str)
elif dest_exists and dest.samefile(src):
# update, hardlink and it already points to the right file, do nothing
@@ -800,11 +809,14 @@ def _export_photo(
# dest.unlink()
fileutil.unlink(dest)
fileutil.hardlink(src, dest)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
update_updated_files.append(dest_str)
exported_files.append(dest_str)
else:
@@ -813,11 +825,14 @@ def _export_photo(
f"Update: exporting new file with export_as_hardlink {src} {dest}"
)
fileutil.hardlink(src, dest)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
exported_files.append(dest_str)
update_new_files.append(dest_str)
else:
@@ -829,12 +844,15 @@ def _export_photo(
fileutil.unlink(dest)
logging.debug(f"Not update: copying file {src} {dest}")
fileutil.copy(src, dest_str, norsrc=no_xattr)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
exported_files.append(dest_str)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
# elif dest_exists and not exiftool and cmp_file(dest_str, export_db.get_stat_orig_for_file(dest_str)):
elif (
dest_exists
@@ -865,22 +883,28 @@ def _export_photo(
# dest.unlink()
fileutil.unlink(dest)
fileutil.copy(src, dest_str, norsrc=no_xattr)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
exported_files.append(dest_str)
update_updated_files.append(dest_str)
else:
# destination doesn't exist, copy the file
logging.debug(f"Update: copying new file {src} {dest}")
fileutil.copy(src, dest_str, norsrc=no_xattr)
export_db.set_uuid_for_file(dest_str, self.uuid)
export_db.set_info_for_uuid(self.uuid, self.json())
export_db.set_stat_orig_for_file(dest_str, fileutil.file_sig(dest_str))
export_db.set_stat_exif_for_file(dest_str, (None, None, None))
export_db.set_exifdata_for_file(dest_str, None)
export_db.set_data(
dest_str,
self.uuid,
fileutil.file_sig(dest_str),
(None, None, None),
self.json(),
None,
)
exported_files.append(dest_str)
update_new_files.append(dest_str)