path: root/rapid/subfolderfile.py

#!/usr/bin/python
# -*- coding: latin1 -*-

### Copyright (C) 2011 Damon Lynch <damonlynch@gmail.com>

### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.

### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.

### You should have received a copy of the GNU General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

"""
Generates names for files and folders.

Runs a daemon process.
"""

import os, datetime, collections, fractions

import gio
import multiprocessing
import logging
logger = multiprocessing.get_logger()


import rpdfile
import rpdmultiprocessing as rpdmp
import generatename as gn
import problemnotification as pn
import prefsrapid
import config

from gettext import gettext as _

class SyncRawJpeg:
    def __init__(self):
        self.photos = {}
        
    def add_download(self, name, extension, date_time, sub_seconds, sequence_number_used):
        if name not in self.photos:
            self.photos[name] = ([extension], date_time, sub_seconds, sequence_number_used)
        else:
            if extension not in self.photos[name][0]:
                self.photos[name][0].append(extension)

        
    def matching_pair(self, name, extension, date_time, sub_seconds):
        """Checks to see if the image matches an image that has already been downloaded.
        Image name (minus extension), exif date time, and exif subseconds are checked.
        
        Returns -1 and a sequence number if the name, extension, and exif values match (i.e. it has already been downloaded)
        Returns 0 and a sequence number if name and exif values match, but the extension is different (i.e. a matching RAW + JPG image)
        Returns -99 and a sequence number of None if photos detected with the same filenames, but taken at different times
        Returns 1 and a sequence number of None if no match"""
        
        if name in self.photos:
            if self.photos[name][1] == date_time and self.photos[name][2] == sub_seconds:
                if extension in self.photos[name][0]:
                    return (-1, self.photos[name][3])
                else:
                    return (0, self.photos[name][3])
            else:
                return (-99, None)
        return (1, None)
        
    def ext_exif_date_time(self, name):
        """Returns first extension, exif date time and subseconds data for the already downloaded photo"""
        return (self.photos[name][0][0], self.photos[name][1], self.photos[name][2])
        
def time_subseconds_human_readable(date, subseconds):
    return _("%(hour)s:%(minute)s:%(second)s:%(subsecond)s") % \
            {'hour':date.strftime("%H"),
             'minute':date.strftime("%M"), 
             'second':date.strftime("%S"),
             'subsecond': subseconds}        

def load_metadata(rpd_file):
    """
    Loads the metadata for the file. Returns True if operation succeeded, false
    otherwise
    """
    if rpd_file.metadata is None:        
        if not rpd_file.load_metadata():
            # Error in reading metadata
            rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized})
            return False
    return True
        

def _generate_name(generator, rpd_file):
    
    do_generation = True
    if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
        do_generation = load_metadata(rpd_file)
    else:
        if rpd_file.metadata is None:
            rpd_file.load_metadata()

    if do_generation:
        value = generator.generate_name(rpd_file)
        if value is None:
            value = ''
    else:
        value = ''
    
    return value

def generate_subfolder(rpd_file):
    
    if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
        generator = gn.PhotoSubfolder(rpd_file.subfolder_pref_list)
    else:
        generator = gn.VideoSubfolder(rpd_file.subfolder_pref_list)
        
    rpd_file.download_subfolder = _generate_name(generator, rpd_file)
    return rpd_file
    
def generate_name(rpd_file):
    do_generation = True
    
    if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
        generator = gn.PhotoName(rpd_file.name_pref_list)
    else:
        generator = gn.VideoName(rpd_file.name_pref_list)
        
    rpd_file.download_name = _generate_name(generator, rpd_file)
    return rpd_file
        

class SubfolderFile(multiprocessing.Process):
    def __init__(self, results_pipe, sequence_values, focal_length):
        multiprocessing.Process.__init__(self)
        self.daemon = True
        self.results_pipe = results_pipe
        
        self.downloads_today = sequence_values[0]
        self.downloads_today_date = sequence_values[1]
        self.day_start = sequence_values[2]
        self.refresh_downloads_today = sequence_values[3]
        self.stored_sequence_no = sequence_values[4]
        self.uses_stored_sequence_no = sequence_values[5]
        self.uses_session_sequece_no = sequence_values[6]
        self.uses_sequence_letter = sequence_values[7]
        
        self.focal_length = focal_length
        
        logger.debug("Start of day is set to %s", self.day_start.value)
        
    def progress_callback_no_update(self, amount_downloaded, total):
        pass
        
    def file_exists(self, rpd_file, identifier=None):
        """
        Notify user that the download file already exists
        """
        # get information on when the existing file was last modified
        try:
            modification_time = os.path.getmtime(rpd_file.download_full_file_name)
            dt = datetime.datetime.fromtimestamp(modification_time)
            date = dt.strftime("%x")
            time = dt.strftime("%X")
        except:
            logger.warning("Could not determine the file modification time of %s", 
                                rpd_file.download_full_file_name)
            date = time = ''
            
        if not identifier:
            rpd_file.add_problem(None, pn.FILE_ALREADY_EXISTS_NO_DOWNLOAD, 
                                {'filetype':rpd_file.title_capitalized})
            rpd_file.add_extra_detail(pn.EXISTING_FILE, 
                                {'filetype': rpd_file.title, 
                                'date': date, 'time': time})
            rpd_file.status = config.STATUS_DOWNLOAD_FAILED
            rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.EXISTING_FILE] % \
                  {'date':date, 'time':time, 'filetype': rpd_file.title}
        else:
            rpd_file.add_problem(None, pn.UNIQUE_IDENTIFIER_ADDED, 
                                {'filetype':rpd_file.title_capitalized})
            rpd_file.add_extra_detail(pn.UNIQUE_IDENTIFIER, 
                                {'identifier': identifier, 
                                'filetype': rpd_file.title,
                                'date': date, 'time': time})
            rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
            rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.UNIQUE_IDENTIFIER] % \
                   {'identifier': identifier, 'filetype': rpd_file.title,
                    'date': date, 'time': time}
        rpd_file.error_title = rpd_file.problem.get_title()
        rpd_file.error_msg = _("Source: %(source)s\nDestination: %(destination)s") \
                % {'source': rpd_file.full_file_name, 
                   'destination': rpd_file.download_full_file_name}
        return rpd_file
        
    def download_failure_file_error(self, rpd_file, inst):
        """
        Handle cases where file failed to download
        """
        rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title})
        rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst)
        rpd_file.status = config.STATUS_DOWNLOAD_FAILED
        logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst)
        
        rpd_file.error_title = rpd_file.problem.get_title()
        rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
                              {'problem': rpd_file.problem.get_problems(),
                               'file': rpd_file.full_file_name}
        
        return rpd_file
        
    def same_name_different_exif(self, sync_photo_name, rpd_file):
        """Notify the user that a file was already downloaded with the same name, but the exif information was different"""
        i1_ext, i1_date_time, i1_subseconds = self.sync_raw_jpeg.ext_exif_date_time(sync_photo_name)
        detail = {'image1': "%s%s" % (sync_photo_name, i1_ext), 
            'image1_date': i1_date_time.strftime("%x"),
            'image1_time': time_subseconds_human_readable(i1_date_time, i1_subseconds), 
            'image2':      rpd_file.name, 
            'image2_date': rpd_file.metadata.date_time().strftime("%x"),
            'image2_time': time_subseconds_human_readable(
                                rpd_file.metadata.date_time(), 
                                rpd_file.metadata.sub_seconds())}
        rpd_file.add_problem(None, pn.SAME_FILE_DIFFERENT_EXIF, detail)

        rpd_file.error_title = _('Photos detected with the same filenames, but taken at different times')
        rpd_file.error_msg = pn.problem_definitions[pn.SAME_FILE_DIFFERENT_EXIF][1] % detail
        rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
        return rpd_file
        
        
    def run(self):
        """
        Get subfolder and name.
        Attempt to move the file from it's temporary directory.
        If successful, increment sequence values.
        Report any success or failure.
        """
        i = 0
        download_count = 0
        
        duplicate_files = {}


        # Track downloads today, using a class whose purpose is to 
        # take the value in the user prefs, increment, and then be used
        # to update the prefs (which can only happen via the main process)
        self.downloads_today_tracker = prefsrapid.DownloadsTodayTracker(
                                        day_start = self.day_start.value,
                                        downloads_today = self.downloads_today.value,
                                        downloads_today_date = self.downloads_today_date.value)
                                                
        # Track sequences using shared downloads today and stored sequence number
        # (shared with main process)
        self.sequences = gn.Sequences(self.downloads_today_tracker, 
                                      self.stored_sequence_no.value)
                                      
        self.sync_raw_jpeg = SyncRawJpeg()
                                      

        while True:
            logger.debug("Finished %s. Getting next task.", download_count)

            task = self.results_pipe.recv()
            
            # rename file and move to generated subfolder                    
            download_succeeded, download_count, rpd_file = task
            
            move_succeeded = False
            

            if download_succeeded:
                temp_file = gio.File(rpd_file.temp_full_file_name)

                synchronize_raw_jpg_failed = False
                if not (rpd_file.synchronize_raw_jpg and
                    rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO):
                    synchronize_raw_jpg = False
                    sequence_to_use = None
                else:
                    synchronize_raw_jpg = True
                    sync_photo_name, sync_photo_ext = os.path.splitext(rpd_file.name)
                    if not load_metadata(rpd_file):
                        synchronize_raw_jpg_failed = True
                    else:
                        j, sequence_to_use = self.sync_raw_jpeg.matching_pair(
                                name=sync_photo_name, extension=sync_photo_ext, 
                                date_time=rpd_file.metadata.date_time(), 
                                sub_seconds=rpd_file.metadata.sub_seconds())
                        if j == -1:
                            # this exact file has already been downloaded (same extension, same filename, and same exif date time subsecond info)
                            if (rpd_file.download_conflict_resolution <>
                                    config.ADD_UNIQUE_IDENTIFIER):
                                rpd_file.add_problem(None, pn.FILE_ALREADY_DOWNLOADED, {'filetype': rpd_file.title_capitalized})
                                rpd_file.error_title = _('Photo has already been downloaded')
                                rpd_file.error_msg = _("Source: %(source)s") % {'source': rpd_file.full_file_name}
                                rpd_file.status = config.STATUS_DOWNLOAD_FAILED
                                synchronize_raw_jpg_failed = True
                        else:
                            self.sequences.set_matched_sequence_value(sequence_to_use)
                            if j == -99:
                                rpd_file = self.same_name_different_exif(sync_photo_name, rpd_file)
                    
                if synchronize_raw_jpg_failed:
                    generation_succeeded = False
                else:
                    # Generate subfolder name and new file name
                    generation_succeeded = True
                    
                    # check to see if focal length and aperture data should be manipulated
                    if self.focal_length is not None and rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
                        if load_metadata(rpd_file):
                            a = rpd_file.metadata.aperture()
                            if a == '0.0':
                                fl = rpd_file.metadata["Exif.Photo.FocalLength"].value
                                logger.info("Samyang lens - adjusting focal length and aperture... ")
                                #~ try:
                                rpd_file.metadata["Exif.Photo.FocalLength"] = fractions.Fraction(14,1)
                                rpd_file.metadata["Exif.Photo.FNumber"] = fractions.Fraction(8,1)
                                    #~ rpd_file.metadata.write(preserve_timestamps=True)
                                #~ logger.info("...wrote new value")
                                #~ except:
                                    #~ logger.error("failed to write value!") 
                            
                        
                    rpd_file = generate_subfolder(rpd_file)
                    if rpd_file.download_subfolder:
                        
                        if self.refresh_downloads_today.value:
                            # overwrite downloads today value tracked here,
                            # as user has modified their preferences
                            self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value)
                            self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value)
                            self.downloads_today_tracker.day_start = self.day_start.value
                            self.refresh_downloads_today.value = False
                            
                        # update whatever the stored value is
                        self.sequences.stored_sequence_no = self.stored_sequence_no.value
                        rpd_file.sequences = self.sequences
                        
                        # generate the file name
                        rpd_file = generate_name(rpd_file)
                        
                        if rpd_file.has_problem():
                            rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
                            rpd_file.error_title = rpd_file.problem.get_title()
                            rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
                                     {'problem': rpd_file.problem.get_problems(),
                                      'file': rpd_file.full_file_name}
                                      
                    # Check for any errors
                    if not rpd_file.download_subfolder or not rpd_file.download_name:
                        if not rpd_file.download_subfolder and not rpd_file.download_name:
                            area = _("subfolder and filename")
                        elif not rpd_file.download_name:
                            area = _("filename")
                        else:
                            area = _("subfolder")
                        rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area})
                        rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area})
                        generation_succeeded = False
                        rpd_file.status = config.STATUS_DOWNLOAD_FAILED
                        
                        rpd_file.error_title = rpd_file.problem.get_title()
                        rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
                                     {'problem': rpd_file.problem.get_problems(),
                                      'file': rpd_file.full_file_name}
                    
                    
                if generation_succeeded:
                    rpd_file.download_path = os.path.join(rpd_file.download_folder, rpd_file.download_subfolder)
                    rpd_file.download_full_file_name = os.path.join(rpd_file.download_path, rpd_file.download_name)
                    
                    subfolder = gio.File(path=rpd_file.download_path)
                    
                    # Create subfolder if it does not exist.
                    # It is possible to skip the query step, and just try to create
                    # the directories and ignore the error of it already existing -
                    # but it takes twice as long to fail with an error than just
                    # run the straight query
                    
                    if not subfolder.query_exists(cancellable=None):
                        try:
                            subfolder.make_directory_with_parents(cancellable=gio.Cancellable())
                        except gio.Error, inst:
                            # The directory may have been created by another process
                            # between the time it takes to query and the time it takes
                            # to create a new directory. Ignore such errors.
                            if inst.code <> gio.ERROR_EXISTS:
                                logger.error("Failed to create download subfolder: %s", rpd_file.download_path)
                                logger.error(inst)
                                rpd_file.error_title = _("Failed to create download subfolder")
                                rpd_file.error_msg = _("Path: %s") % rpd_file.download_path
                    
                    # Move temp file to subfolder

                    download_file = gio.File(rpd_file.download_full_file_name)
                    
                    add_unique_identifier = False
                    try:
                        temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
                        move_succeeded = True
                        if rpd_file.status <> config.STATUS_DOWNLOADED_WITH_WARNING:
                            rpd_file.status = config.STATUS_DOWNLOADED
                    except gio.Error, inst:
                        if inst.code == gio.ERROR_EXISTS:
                            if (rpd_file.download_conflict_resolution == 
                                config.ADD_UNIQUE_IDENTIFIER):
                                add_unique_identifier = True
                            else:
                                rpd_file = self.file_exists(rpd_file)
                        else:
                            rpd_file = self.download_failure_file_error(rpd_file, inst)
                    
                    if add_unique_identifier:
                        name = os.path.splitext(rpd_file.download_name)
                        full_name = rpd_file.download_full_file_name
                        suffix_already_used = True
                        while suffix_already_used:
                            duplicate_files[full_name] = duplicate_files.get(
                                                              full_name, 0) + 1
                            identifier = '_%s' % duplicate_files[full_name]
                            rpd_file.download_name = name[0] + identifier + name[1]
                            rpd_file.download_full_file_name = os.path.join(
                                                    rpd_file.download_path,
                                                    rpd_file.download_name)
                            download_file = gio.File(
                                            rpd_file.download_full_file_name)
                            
                            try:
                                temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
                                move_succeeded = True
                                suffix_already_used = False
                                rpd_file = self.file_exists(rpd_file, identifier)
                                logger.error("%s: %s - %s", rpd_file.full_file_name, 
                                    rpd_file.problem.get_title(), 
                                    rpd_file.problem.get_problems())
                            except gio.Error, inst:
                                if inst.code <> gio.ERROR_EXISTS:
                                    rpd_file = self.download_failure_file_error(rpd_file, inst)
                            
                        
                        
                    logger.debug("Finish processing file: %s", download_count)                    
                
                if move_succeeded:
                    if synchronize_raw_jpg:
                        if sequence_to_use is None:
                            sequence = self.sequences.create_matched_sequences()
                        else:
                            sequence = sequence_to_use
                        self.sync_raw_jpeg.add_download(name=sync_photo_name,
                                extension=sync_photo_ext,
                                date_time=rpd_file.metadata.date_time(),
                                sub_seconds=rpd_file.metadata.sub_seconds(),
                                sequence_number_used=sequence)
                    if sequence_to_use is None:
                        if self.uses_session_sequece_no.value or self.uses_sequence_letter.value:
                            self.sequences.increment(
                                            self.uses_session_sequece_no.value, 
                                            self.uses_sequence_letter.value)
                        if self.uses_stored_sequence_no.value:
                            self.stored_sequence_no.value += 1
                        self.downloads_today_tracker.increment_downloads_today()
                        self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today()
                        self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date()
                
                if not move_succeeded:
                    logger.error("%s: %s - %s", rpd_file.full_file_name, 
                                 rpd_file.problem.get_title(), 
                                 rpd_file.problem.get_problems())
                    try:
                        temp_file.delete(cancellable=None)
                    except gio.Error, inst:
                        logger.error("Failed to delete temporary file %s", rpd_file.temp_full_file_name)
                        logger.error(inst)
                    

                    
            
            
            rpd_file.metadata = None #purge metadata, as it cannot be pickled
            rpd_file.sequences = None
            self.results_pipe.send((move_succeeded, rpd_file,))
            
            i += 1