path: root/rapid/copyfiles.py

#!/usr/bin/python
# -*- coding: latin1 -*-

### Copyright (C) 2011-2014 Damon Lynch <damonlynch@gmail.com>

### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.

### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.

### You should have received a copy of the GNU General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
### USA

import multiprocessing
import tempfile
import os
import random
import string

import logging
logger = multiprocessing.get_logger()

import rpdmultiprocessing as rpdmp
import rpdfile
import problemnotification as pn
import config
import thumbnail as tn
import io
import shutil
import stat
import hashlib

from gettext import gettext as _

def copy_file_metadata(src, dst, logger=None):
    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.

    Adapated from python's shutil.copystat.

    Necessary because with some NTFS file systems, there can be problems
    with setting filesystem metadata like permissions and modification time"""

    st = os.stat(src)
    mode = stat.S_IMODE(st.st_mode)
    try:
        os.utime(dst, (st.st_atime, st.st_mtime))
    except OSError as inst:
        if logger:
            logger.warning("Couldn't adjust file modification time when copying %s. %s: %s", src, inst.errno, inst.strerror)
    try:
        os.chmod(dst, mode)
    except OSError as inst:
        if logger:
            logger.warning("Couldn't adjust file permissions when copying %s. %s: %s", src, inst.errno, inst.strerror)

    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
        try:
            os.chflags(dst, st.st_flags)
        except OSError as inst:
            for err in 'EOPNOTSUPP', 'ENOTSUP':
                if hasattr(errno, err) and inst.errno == getattr(errno, err):
                    break
            else:
                raise


class CopyFiles(multiprocessing.Process):
    """
    Copies files from source to temporary directory, giving them a random name
    """
    def __init__(self, photo_download_folder, video_download_folder, files,
                 verify_file,
                 modify_files_during_download, modify_pipe,
                 scan_pid,
                 batch_size_MB, results_pipe, terminate_queue,
                 run_event):
        multiprocessing.Process.__init__(self)
        self.results_pipe = results_pipe
        self.terminate_queue = terminate_queue
        self.batch_size_bytes = batch_size_MB * 1048576 # * 1024 * 1024
        self.io_buffer = 1048576
        self.photo_download_folder = photo_download_folder
        self.video_download_folder = video_download_folder
        self.files = files
        self.verify_file = verify_file
        self.modify_files_during_download = modify_files_during_download
        self.modify_pipe = modify_pipe
        self.scan_pid = scan_pid
        self.no_files= len(self.files)
        self.run_event = run_event


    def check_termination_request(self):
        """
        Check to see this process has not been requested to immediately terminate
        """
        if not self.terminate_queue.empty():
            x = self.terminate_queue.get()
            # terminate immediately
            logger.info("Terminating file copying")
            return True
        return False


    def update_progress(self, amount_downloaded, total):

        chunk_downloaded = amount_downloaded - self.bytes_downloaded
        if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
            self.bytes_downloaded = amount_downloaded
            self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
            if amount_downloaded == total:
                self.bytes_downloaded = 0


    def run(self):
        """start the actual copying of files"""

        #characters used to generate temporary filenames
        filename_characters = string.letters + string.digits

        self.bytes_downloaded = 0
        self.total_downloaded = 0

        self.create_temp_dirs()

        # Send the location of both temporary directories, so they can be
        # removed once another process attempts to rename all the files in them
        # and move them to generated subfolders
        self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_TEMP_DIRS,
                                                     (self.scan_pid,
                                                     self.photo_temp_dir,
                                                     self.video_temp_dir))))

        if self.photo_temp_dir or self.video_temp_dir:

            self.thumbnail_maker = tn.Thumbnail()

            for i in range(self.no_files):
                rpd_file = self.files[i]

                # pause if instructed by the caller
                self.run_event.wait()

                if self.check_termination_request():
                    return None

                #generate temporary name 5 digits long, no extension
                temp_name = ''.join(random.choice(filename_characters) for i in xrange(5))

                temp_full_file_name = os.path.join(
                                    self._get_dest_dir(rpd_file.file_type),
                                    temp_name)
                rpd_file.temp_full_file_name = temp_full_file_name

                copy_succeeded = False

                src_bytes = ''

                try:
                    dest = io.open(temp_full_file_name, 'wb', self.io_buffer)
                    src = io.open(rpd_file.full_file_name, 'rb', self.io_buffer)
                    total = rpd_file.size
                    amount_downloaded = 0
                    while True:
                        # first check if process is being terminated
                        if self.check_termination_request():
                            logger.debug("Closing partially written temporary file")
                            dest.close()
                            src.close()
                            return None
                        else:
                            chunk = src.read(self.io_buffer)
                            if chunk:
                                dest.write(chunk)
                                src_bytes += chunk
                                amount_downloaded += len(chunk)
                                self.update_progress(amount_downloaded, total)
                            else:
                                break
                    dest.close()
                    src.close()
                    copy_succeeded = True
                except (IOError, OSError) as inst:
                    rpd_file.add_problem(None,
                        pn.DOWNLOAD_COPYING_ERROR_W_NO,
                        {'filetype': rpd_file.title})
                    rpd_file.add_extra_detail(
                        pn.DOWNLOAD_COPYING_ERROR_W_NO_DETAIL,
                        {'errorno': inst.errno, 'strerror': inst.strerror})

                    rpd_file.status = config.STATUS_DOWNLOAD_FAILED

                    rpd_file.error_title = rpd_file.problem.get_title()
                    rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
                                  {'problem': rpd_file.problem.get_problems(),
                                   'file': rpd_file.full_file_name}

                    logger.error("Failed to download file: %s", rpd_file.full_file_name)
                    logger.error(inst)
                    self.update_progress(rpd_file.size, rpd_file.size)
                except:
                    rpd_file.add_problem(None,
                        pn.DOWNLOAD_COPYING_ERROR,
                        {'filetype': rpd_file.title})
                    rpd_file.add_extra_detail(
                        pn.DOWNLOAD_COPYING_ERROR_DETAIL,
                        _("An unknown error occurred"))

                    rpd_file.status = config.STATUS_DOWNLOAD_FAILED

                    rpd_file.error_title = rpd_file.problem.get_title()
                    rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
                                  {'problem': rpd_file.problem.get_problems(),
                                   'file': rpd_file.full_file_name}

                    logger.error("Failed to download file: %s", rpd_file.full_file_name)
                    self.update_progress(rpd_file.size, rpd_file.size)

                # increment this amount regardless of whether the copy actually
                # succeeded or not. It's neccessary to keep the user informed.
                self.total_downloaded += rpd_file.size

                try:
                    copy_file_metadata(rpd_file.full_file_name, temp_full_file_name, logger)
                except:
                    logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.full_file_name)

                # copy THM (video thumbnail file) if there is one
                if copy_succeeded and rpd_file.thm_full_name:
                    # reuse video's file name
                    temp_thm_full_name = temp_full_file_name + '__rpd__thm'
                    try:
                        shutil.copyfile(rpd_file.thm_full_name, temp_thm_full_name)
                        rpd_file.temp_thm_full_name = temp_thm_full_name
                        logger.debug("Copied video THM file %s", rpd_file.temp_thm_full_name)
                    except (IOError, OSError) as inst:
                        logger.error("Failed to download video THM file: %s", rpd_file.thm_full_name)
                        logger.error("%s: %s", inst.errno, inst.strerror)
                    try:
                        copy_file_metadata(rpd_file.thm_full_name, temp_thm_full_name. logger)
                    except:
                        logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.thm_full_name)

                else:
                    temp_thm_full_name = None

                #copy audio file if there is one
                if copy_succeeded and rpd_file.audio_file_full_name:
                    # reuse photo's file name
                    temp_audio_full_name = temp_full_file_name + '__rpd__audio'
                    try:
                        shutil.copyfile(rpd_file.audio_file_full_name, temp_audio_full_name)
                        rpd_file.temp_audio_full_name = temp_audio_full_name
                        logger.debug("Copied audio file %s", rpd_file.temp_audio_full_name)
                    except (IOError, OSError) as inst:
                        logger.error("Failed to download audio file: %s", rpd_file.audio_file_full_name)
                        logger.error("%s: %s", inst.errno, inst.strerror)
                    try:
                        copy_file_metadata(rpd_file.audio_file_full_name, temp_audio_full_name, logger)
                    except:
                        logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.audio_file_full_name)



                if copy_succeeded and rpd_file.generate_thumbnail:
                    thumbnail, thumbnail_icon = self.thumbnail_maker.get_thumbnail(
                                    temp_full_file_name,
                                    temp_thm_full_name,
                                    rpd_file.file_type,
                                    (160, 120), (100,100))
                else:
                    thumbnail = None
                    thumbnail_icon = None

                if copy_succeeded and self.verify_file:
                    rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()

                if rpd_file.metadata is not None:
                    rpd_file.metadata = None


                download_count = i + 1
                if self.modify_files_during_download and copy_succeeded:
                    copy_finished = download_count == self.no_files

                    self.modify_pipe.send((rpd_file, download_count, temp_full_file_name,
                        thumbnail_icon, thumbnail, copy_finished))
                else:
                    self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_FILE,
                        (copy_succeeded, rpd_file, download_count,
                         temp_full_file_name,
                         thumbnail_icon, thumbnail))))


        self.results_pipe.send((rpdmp.CONN_COMPLETE, None))


    def _get_dest_dir(self, file_type):
        if file_type == rpdfile.FILE_TYPE_PHOTO:
            return self.photo_temp_dir
        else:
            return self.video_temp_dir

    def _create_temp_dir(self, folder):
        try:
            temp_dir = tempfile.mkdtemp(prefix="rpd-tmp-", dir=folder)
        except OSError, (errno, strerror):
            # FIXME: error reporting
            logger.error("Failed to create temporary directory in %s: %s %s",
                         errono,
                         strerror,
                         folder)
            temp_dir = None

        return temp_dir

    def create_temp_dirs(self):
        self.photo_temp_dir = self.video_temp_dir = None
        if self.photo_download_folder is not None:
            self.photo_temp_dir = self._create_temp_dir(self.photo_download_folder)
            logger.debug("Photo temporary directory: %s", self.photo_temp_dir)
        if self.video_download_folder is not None:
            self.video_temp_dir = self._create_temp_dir(self.video_download_folder)
            logger.debug("Video temporary directory: %s", self.video_temp_dir)