summaryrefslogtreecommitdiff
path: root/rapid/subfolderfile.py
diff options
context:
space:
mode:
authorJulien Valroff <julien@kirya.net>2011-04-08 07:09:55 +0200
committerJulien Valroff <julien@kirya.net>2011-04-08 07:09:55 +0200
commitc7dff12ef94a1f5e5faa422444ebd2cf47b6b549 (patch)
tree028059e3a81327c18ec61f46bd30a3ed82a707ce /rapid/subfolderfile.py
parent4e2b25f9d85f23c51ae51a03049436a6778da69e (diff)
parenteb4c5cc4472b16ce10401611140381e5ba5b6aca (diff)
Merge commit 'upstream/0.4.0_alpha4' into experimental
Diffstat (limited to 'rapid/subfolderfile.py')
-rw-r--r--rapid/subfolderfile.py323
1 files changed, 267 insertions, 56 deletions
diff --git a/rapid/subfolderfile.py b/rapid/subfolderfile.py
index 53118c2..c55b6d8 100644
--- a/rapid/subfolderfile.py
+++ b/rapid/subfolderfile.py
@@ -23,7 +23,7 @@ Generates names for files and folders.
Runs a daemon process.
"""
-import os
+import os, datetime, collections
import gio
import multiprocessing
@@ -40,17 +40,66 @@ import config
from gettext import gettext as _
+class SyncRawJpeg:
+ def __init__(self):
+ self.photos = {}
+
+ def add_download(self, name, extension, date_time, sub_seconds, sequence_number_used):
+ if name not in self.photos:
+ self.photos[name] = ([extension], date_time, sub_seconds, sequence_number_used)
+ else:
+ if extension not in self.photos[name][0]:
+ self.photos[name][0].append(extension)
+
+
+ def matching_pair(self, name, extension, date_time, sub_seconds):
+ """Checks to see if the image matches an image that has already been downloaded.
+ Image name (minus extension), exif date time, and exif subseconds are checked.
+
+ Returns -1 and a sequence number if the name, extension, and exif values match (i.e. it has already been downloaded)
+ Returns 0 and a sequence number if name and exif values match, but the extension is different (i.e. a matching RAW + JPG image)
+ Returns -99 and a sequence number of None if photos detected with the same filenames, but taken at different times
+ Returns 1 and a sequence number of None if no match"""
+
+ if name in self.photos:
+ if self.photos[name][1] == date_time and self.photos[name][2] == sub_seconds:
+ if extension in self.photos[name][0]:
+ return (-1, self.photos[name][3])
+ else:
+ return (0, self.photos[name][3])
+ else:
+ return (-99, None)
+ return (1, None)
+
+ def ext_exif_date_time(self, name):
+ """Returns first extension, exif date time and subseconds data for the already downloaded photo"""
+ return (self.photos[name][0][0], self.photos[name][1], self.photos[name][2])
+
+def time_subseconds_human_readable(date, subseconds):
+ return _("%(hour)s:%(minute)s:%(second)s:%(subsecond)s") % \
+ {'hour':date.strftime("%H"),
+ 'minute':date.strftime("%M"),
+ 'second':date.strftime("%S"),
+ 'subsecond': subseconds}
+def load_metadata(rpd_file):
+ """
+ Loads the metadata for the file. Returns True if operation succeeded, false
+ otherwise
+ """
+ if rpd_file.metadata is None:
+ if not rpd_file.load_metadata():
+ # Error in reading metadata
+ rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized})
+ return False
+ return True
+
def _generate_name(generator, rpd_file):
do_generation = True
if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
- if rpd_file.metadata is None:
- if not rpd_file.load_metadata():
- # Error in reading metadata
- rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized})
- do_generation = False
+ do_generation = load_metadata(rpd_file)
else:
if rpd_file.metadata is None:
rpd_file.load_metadata()
@@ -103,12 +152,85 @@ class SubfolderFile(multiprocessing.Process):
logger.debug("Start of day is set to %s", self.day_start.value)
-
-
-
def progress_callback_no_update(self, amount_downloaded, total):
pass
+ def file_exists(self, rpd_file, identifier=None):
+ """
+ Notify user that the download file already exists
+ """
+ # get information on when the existing file was last modified
+ try:
+ modification_time = os.path.getmtime(rpd_file.download_full_file_name)
+ dt = datetime.datetime.fromtimestamp(modification_time)
+ date = dt.strftime("%x")
+ time = dt.strftime("%X")
+ except:
+ logger.warning("Could not determine the file modification time of %s",
+ rpd_file.download_full_file_name)
+ date = time = ''
+
+ if not identifier:
+ rpd_file.add_problem(None, pn.FILE_ALREADY_EXISTS_NO_DOWNLOAD,
+ {'filetype':rpd_file.title_capitalized})
+ rpd_file.add_extra_detail(pn.EXISTING_FILE,
+ {'filetype': rpd_file.title,
+ 'date': date, 'time': time})
+ rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+ rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.EXISTING_FILE] % \
+ {'date':date, 'time':time, 'filetype': rpd_file.title}
+ else:
+ rpd_file.add_problem(None, pn.UNIQUE_IDENTIFIER_ADDED,
+ {'filetype':rpd_file.title_capitalized})
+ rpd_file.add_extra_detail(pn.UNIQUE_IDENTIFIER,
+ {'identifier': identifier,
+ 'filetype': rpd_file.title,
+ 'date': date, 'time': time})
+ rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+ rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.UNIQUE_IDENTIFIER] % \
+ {'identifier': identifier, 'filetype': rpd_file.title,
+ 'date': date, 'time': time}
+ rpd_file.error_title = rpd_file.problem.get_title()
+ rpd_file.error_msg = _("Source: %(source)s\nDestination: %(destination)s") \
+ % {'source': rpd_file.full_file_name,
+ 'destination': rpd_file.download_full_file_name}
+ return rpd_file
+
+ def download_failure_file_error(self, rpd_file, inst):
+ """
+ Handle cases where file failed to download
+ """
+ rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title})
+ rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst)
+ rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+ logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst)
+
+ rpd_file.error_title = rpd_file.problem.get_title()
+ rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+ {'problem': rpd_file.problem.get_problems(),
+ 'file': rpd_file.full_file_name}
+
+ return rpd_file
+
+ def same_name_different_exif(self, sync_photo_name, rpd_file):
+ """Notify the user that a file was already downloaded with the same name, but the exif information was different"""
+ i1_ext, i1_date_time, i1_subseconds = self.sync_raw_jpeg.ext_exif_date_time(sync_photo_name)
+ detail = {'image1': "%s%s" % (sync_photo_name, i1_ext),
+ 'image1_date': i1_date_time.strftime("%x"),
+ 'image1_time': time_subseconds_human_readable(i1_date_time, i1_subseconds),
+ 'image2': rpd_file.name,
+ 'image2_date': rpd_file.metadata.date_time().strftime("%x"),
+ 'image2_time': time_subseconds_human_readable(
+ rpd_file.metadata.date_time(),
+ rpd_file.metadata.sub_seconds())}
+ rpd_file.add_problem(None, pn.SAME_FILE_DIFFERENT_EXIF, detail)
+
+ rpd_file.error_title = _('Photos detected with the same filenames, but taken at different times')
+ rpd_file.error_msg = pn.problem_definitions[pn.SAME_FILE_DIFFERENT_EXIF][1] % detail
+ rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+ return rpd_file
+
+
def run(self):
"""
Get subfolder and name.
@@ -118,6 +240,8 @@ class SubfolderFile(multiprocessing.Process):
"""
i = 0
download_count = 0
+
+ duplicate_files = {}
# Track downloads today, using a class whose purpose is to
@@ -133,6 +257,8 @@ class SubfolderFile(multiprocessing.Process):
self.sequences = gn.Sequences(self.downloads_today_tracker,
self.stored_sequence_no.value)
+ self.sync_raw_jpeg = SyncRawJpeg()
+
while True:
logger.debug("Finished %s. Getting next task.", download_count)
@@ -148,42 +274,79 @@ class SubfolderFile(multiprocessing.Process):
if download_succeeded:
temp_file = gio.File(rpd_file.temp_full_file_name)
- # Generate subfolder name and new file name
- generation_succeeded = True
- rpd_file = generate_subfolder(rpd_file)
- if rpd_file.download_subfolder:
+ synchronize_raw_jpg_failed = False
+ if not (rpd_file.synchronize_raw_jpg and
+ rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO):
+ synchronize_raw_jpg = False
+ sequence_to_use = None
+ else:
+ synchronize_raw_jpg = True
+ sync_photo_name, sync_photo_ext = os.path.splitext(rpd_file.name)
+ if not load_metadata(rpd_file):
+ synchronize_raw_jpg_failed = True
+ else:
+ j, sequence_to_use = self.sync_raw_jpeg.matching_pair(
+ name=sync_photo_name, extension=sync_photo_ext,
+ date_time=rpd_file.metadata.date_time(),
+ sub_seconds=rpd_file.metadata.sub_seconds())
+ if j == -1:
+ # this exact file has already been downloaded (same extension, same filename, and same exif date time subsecond info)
+ if (rpd_file.download_conflict_resolution <>
+ config.ADD_UNIQUE_IDENTIFIER):
+ rpd_file.add_problem(None, pn.FILE_ALREADY_DOWNLOADED, {'filetype': rpd_file.title_capitalized})
+ rpd_file.error_title = _('Photo has already been downloaded')
+ rpd_file.error_msg = _("Source: %(source)s") % {'source': rpd_file.full_file_name}
+ rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+ synchronize_raw_jpg_failed = True
+ else:
+ self.sequences.set_matched_sequence_value(sequence_to_use)
+ if j == -99:
+ rpd_file = self.same_name_different_exif(sync_photo_name, rpd_file)
- if self.refresh_downloads_today.value:
- # overwrite downloads today value tracked here,
- # as user has modified their preferences
- self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value)
- self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value)
- self.downloads_today_tracker.day_start = self.day_start.value
- self.refresh_downloads_today.value = False
+ if synchronize_raw_jpg_failed:
+ generation_succeeded = False
+ else:
+ # Generate subfolder name and new file name
+ generation_succeeded = True
+ rpd_file = generate_subfolder(rpd_file)
+ if rpd_file.download_subfolder:
- # update whatever the stored value is
- self.sequences.stored_sequence_no = self.stored_sequence_no.value
- rpd_file.sequences = self.sequences
-
- # generate the file name
- rpd_file = generate_name(rpd_file)
-
- if rpd_file.has_problem():
- rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+ if self.refresh_downloads_today.value:
+ # overwrite downloads today value tracked here,
+ # as user has modified their preferences
+ self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value)
+ self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value)
+ self.downloads_today_tracker.day_start = self.day_start.value
+ self.refresh_downloads_today.value = False
+
+ # update whatever the stored value is
+ self.sequences.stored_sequence_no = self.stored_sequence_no.value
+ rpd_file.sequences = self.sequences
+
+ # generate the file name
+ rpd_file = generate_name(rpd_file)
+
+ if rpd_file.has_problem():
+ rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
- # Check for any errors
- if not rpd_file.download_subfolder or not rpd_file.download_name:
- if not rpd_file.download_subfolder and not rpd_file.download_name:
- area = _("subfolder and filename")
- elif not rpd_file.download_name:
- area = _("filename")
- else:
- area = _("subfolder")
- rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area})
- rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area})
- generation_succeeded = False
- rpd_file.status = config.STATUS_DOWNLOAD_FAILED
- # FIXME: log error
+ # Check for any errors
+ if not rpd_file.download_subfolder or not rpd_file.download_name:
+ if not rpd_file.download_subfolder and not rpd_file.download_name:
+ area = _("subfolder and filename")
+ elif not rpd_file.download_name:
+ area = _("filename")
+ else:
+ area = _("subfolder")
+ rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area})
+ rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area})
+ generation_succeeded = False
+ rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+
+ rpd_file.error_title = rpd_file.problem.get_title()
+ rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+ {'problem': rpd_file.problem.get_problems(),
+ 'file': rpd_file.full_file_name}
+
if generation_succeeded:
rpd_file.download_path = os.path.join(rpd_file.download_folder, rpd_file.download_subfolder)
@@ -205,36 +368,83 @@ class SubfolderFile(multiprocessing.Process):
# between the time it takes to query and the time it takes
# to create a new directory. Ignore such errors.
if inst.code <> gio.ERROR_EXISTS:
- logger.error("Failed to create directory: %s", rpd_file.download_path)
+ logger.error("Failed to create download subfolder: %s", rpd_file.download_path)
logger.error(inst)
+ rpd_file.error_title = _("Failed to create download subfolder")
+ rpd_file.error_msg = _("Path: %s") % rpd_file.download_path
# Move temp file to subfolder
download_file = gio.File(rpd_file.download_full_file_name)
+ add_unique_identifier = False
try:
temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
move_succeeded = True
if rpd_file.status <> config.STATUS_DOWNLOADED_WITH_WARNING:
rpd_file.status = config.STATUS_DOWNLOADED
except gio.Error, inst:
- rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title})
- rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst)
- rpd_file.status = config.STATUS_DOWNLOAD_FAILED
- logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst)
+ if inst.code == gio.ERROR_EXISTS:
+ if (rpd_file.download_conflict_resolution ==
+ config.ADD_UNIQUE_IDENTIFIER):
+ add_unique_identifier = True
+ else:
+ rpd_file = self.file_exists(rpd_file)
+ else:
+ rpd_file = self.download_failure_file_error(rpd_file, inst)
+
+ if add_unique_identifier:
+ name = os.path.splitext(rpd_file.download_name)
+ full_name = rpd_file.download_full_file_name
+ suffix_already_used = True
+ while suffix_already_used:
+ duplicate_files[full_name] = duplicate_files.get(
+ full_name, 0) + 1
+ identifier = '_%s' % duplicate_files[full_name]
+ rpd_file.download_name = name[0] + identifier + name[1]
+ rpd_file.download_full_file_name = os.path.join(
+ rpd_file.download_path,
+ rpd_file.download_name)
+ download_file = gio.File(
+ rpd_file.download_full_file_name)
+
+ try:
+ temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
+ move_succeeded = True
+ suffix_already_used = False
+ rpd_file = self.file_exists(rpd_file, identifier)
+ logger.error("%s: %s - %s", rpd_file.full_file_name,
+ rpd_file.problem.get_title(),
+ rpd_file.problem.get_problems())
+ except gio.Error, inst:
+ if inst.code <> gio.ERROR_EXISTS:
+ rpd_file = self.download_failure_file_error(rpd_file, inst)
+
+
logger.debug("Finish processing file: %s", download_count)
if move_succeeded:
- if self.uses_session_sequece_no.value or self.uses_sequence_letter.value:
- self.sequences.increment(
- self.uses_session_sequece_no.value,
- self.uses_sequence_letter.value)
- if self.uses_stored_sequence_no.value:
- self.stored_sequence_no.value += 1
- self.downloads_today_tracker.increment_downloads_today()
- self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today()
- self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date()
+ if synchronize_raw_jpg:
+ if sequence_to_use is None:
+ sequence = self.sequences.create_matched_sequences()
+ else:
+ sequence = sequence_to_use
+ self.sync_raw_jpeg.add_download(name=sync_photo_name,
+ extension=sync_photo_ext,
+ date_time=rpd_file.metadata.date_time(),
+ sub_seconds=rpd_file.metadata.sub_seconds(),
+ sequence_number_used=sequence)
+ if sequence_to_use is None:
+ if self.uses_session_sequece_no.value or self.uses_sequence_letter.value:
+ self.sequences.increment(
+ self.uses_session_sequece_no.value,
+ self.uses_sequence_letter.value)
+ if self.uses_stored_sequence_no.value:
+ self.stored_sequence_no.value += 1
+ self.downloads_today_tracker.increment_downloads_today()
+ self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today()
+ self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date()
if not move_succeeded:
logger.error("%s: %s - %s", rpd_file.full_file_name,
@@ -251,6 +461,7 @@ class SubfolderFile(multiprocessing.Process):
rpd_file.metadata = None #purge metadata, as it cannot be pickled
+ rpd_file.sequences = None
self.results_pipe.send((move_succeeded, rpd_file,))
i += 1