summaryrefslogtreecommitdiff
path: root/rapid/copyfiles.py
diff options
context:
space:
mode:
authorJörg Frings-Fürst <debian@jff-webhosting.net>2017-07-06 22:55:08 +0200
committerJörg Frings-Fürst <debian@jff-webhosting.net>2017-07-06 22:55:08 +0200
commit083849161f075878e4175cd03cb7afa83d64e7f5 (patch)
tree101feb02f6306f8f8b335faa39d74f1eaafc8d54 /rapid/copyfiles.py
parentb5287ed17bda10877d84ba86fcf148ee74b93b9b (diff)
New upstream version 0.9.0upstream/0.9.0
Diffstat (limited to 'rapid/copyfiles.py')
-rw-r--r--rapid/copyfiles.py334
1 files changed, 0 insertions, 334 deletions
diff --git a/rapid/copyfiles.py b/rapid/copyfiles.py
deleted file mode 100644
index c63e60f..0000000
--- a/rapid/copyfiles.py
+++ /dev/null
@@ -1,334 +0,0 @@
-#!/usr/bin/python
-# -*- coding: latin1 -*-
-
-### Copyright (C) 2011-2014 Damon Lynch <damonlynch@gmail.com>
-
-### This program is free software; you can redistribute it and/or modify
-### it under the terms of the GNU General Public License as published by
-### the Free Software Foundation; either version 2 of the License, or
-### (at your option) any later version.
-
-### This program is distributed in the hope that it will be useful,
-### but WITHOUT ANY WARRANTY; without even the implied warranty of
-### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-### GNU General Public License for more details.
-
-### You should have received a copy of the GNU General Public License
-### along with this program; if not, write to the Free Software
-### Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
-### USA
-
-import multiprocessing
-import tempfile
-import os
-import random
-import string
-
-import logging
-logger = multiprocessing.get_logger()
-
-import rpdmultiprocessing as rpdmp
-import rpdfile
-import problemnotification as pn
-import config
-import thumbnail as tn
-import io
-import shutil
-import stat
-import hashlib
-
-from gettext import gettext as _
-
-def copy_file_metadata(src, dst, logger=None):
- """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.
-
- Adapated from python's shutil.copystat.
-
- Necessary because with some NTFS file systems, there can be problems
- with setting filesystem metadata like permissions and modification time"""
-
- st = os.stat(src)
- mode = stat.S_IMODE(st.st_mode)
- try:
- os.utime(dst, (st.st_atime, st.st_mtime))
- except OSError as inst:
- if logger:
- logger.warning("Couldn't adjust file modification time when copying %s. %s: %s", src, inst.errno, inst.strerror)
- try:
- os.chmod(dst, mode)
- except OSError as inst:
- if logger:
- logger.warning("Couldn't adjust file permissions when copying %s. %s: %s", src, inst.errno, inst.strerror)
-
- if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
- try:
- os.chflags(dst, st.st_flags)
- except OSError as inst:
- for err in 'EOPNOTSUPP', 'ENOTSUP':
- if hasattr(errno, err) and inst.errno == getattr(errno, err):
- break
- else:
- raise
-
-
-class CopyFiles(multiprocessing.Process):
- """
- Copies files from source to temporary directory, giving them a random name
- """
- def __init__(self, photo_download_folder, video_download_folder, files,
- verify_file,
- modify_files_during_download, modify_pipe,
- scan_pid,
- batch_size_MB, results_pipe, terminate_queue,
- run_event):
- multiprocessing.Process.__init__(self)
- self.results_pipe = results_pipe
- self.terminate_queue = terminate_queue
- self.batch_size_bytes = batch_size_MB * 1048576 # * 1024 * 1024
- self.io_buffer = 1048576
- self.photo_download_folder = photo_download_folder
- self.video_download_folder = video_download_folder
- self.files = files
- self.verify_file = verify_file
- self.modify_files_during_download = modify_files_during_download
- self.modify_pipe = modify_pipe
- self.scan_pid = scan_pid
- self.no_files= len(self.files)
- self.run_event = run_event
-
-
- def check_termination_request(self):
- """
- Check to see this process has not been requested to immediately terminate
- """
- if not self.terminate_queue.empty():
- x = self.terminate_queue.get()
- # terminate immediately
- logger.info("Terminating file copying")
- return True
- return False
-
-
- def update_progress(self, amount_downloaded, total):
-
- chunk_downloaded = amount_downloaded - self.bytes_downloaded
- if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
- self.bytes_downloaded = amount_downloaded
- self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
- if amount_downloaded == total:
- self.bytes_downloaded = 0
-
-
- def run(self):
- """start the actual copying of files"""
-
- #characters used to generate temporary filenames
- filename_characters = string.letters + string.digits
-
- self.bytes_downloaded = 0
- self.total_downloaded = 0
-
- self.create_temp_dirs()
-
- # Send the location of both temporary directories, so they can be
- # removed once another process attempts to rename all the files in them
- # and move them to generated subfolders
- self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_TEMP_DIRS,
- (self.scan_pid,
- self.photo_temp_dir,
- self.video_temp_dir))))
-
- if self.photo_temp_dir or self.video_temp_dir:
-
- self.thumbnail_maker = tn.Thumbnail()
-
- for i in range(self.no_files):
- rpd_file = self.files[i]
-
- # pause if instructed by the caller
- self.run_event.wait()
-
- if self.check_termination_request():
- return None
-
- #generate temporary name 5 digits long, no extension
- temp_name = ''.join(random.choice(filename_characters) for i in xrange(5))
-
- temp_full_file_name = os.path.join(
- self._get_dest_dir(rpd_file.file_type),
- temp_name)
- rpd_file.temp_full_file_name = temp_full_file_name
-
- copy_succeeded = False
-
- src_bytes = ''
-
- try:
- dest = io.open(temp_full_file_name, 'wb', self.io_buffer)
- src = io.open(rpd_file.full_file_name, 'rb', self.io_buffer)
- total = rpd_file.size
- amount_downloaded = 0
- while True:
- # first check if process is being terminated
- if self.check_termination_request():
- logger.debug("Closing partially written temporary file")
- dest.close()
- src.close()
- return None
- else:
- chunk = src.read(self.io_buffer)
- if chunk:
- dest.write(chunk)
- src_bytes += chunk
- amount_downloaded += len(chunk)
- self.update_progress(amount_downloaded, total)
- else:
- break
- dest.close()
- src.close()
- copy_succeeded = True
- except (IOError, OSError) as inst:
- rpd_file.add_problem(None,
- pn.DOWNLOAD_COPYING_ERROR_W_NO,
- {'filetype': rpd_file.title})
- rpd_file.add_extra_detail(
- pn.DOWNLOAD_COPYING_ERROR_W_NO_DETAIL,
- {'errorno': inst.errno, 'strerror': inst.strerror})
-
- rpd_file.status = config.STATUS_DOWNLOAD_FAILED
-
- rpd_file.error_title = rpd_file.problem.get_title()
- rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
- {'problem': rpd_file.problem.get_problems(),
- 'file': rpd_file.full_file_name}
-
- logger.error("Failed to download file: %s", rpd_file.full_file_name)
- logger.error(inst)
- self.update_progress(rpd_file.size, rpd_file.size)
- except:
- rpd_file.add_problem(None,
- pn.DOWNLOAD_COPYING_ERROR,
- {'filetype': rpd_file.title})
- rpd_file.add_extra_detail(
- pn.DOWNLOAD_COPYING_ERROR_DETAIL,
- _("An unknown error occurred"))
-
- rpd_file.status = config.STATUS_DOWNLOAD_FAILED
-
- rpd_file.error_title = rpd_file.problem.get_title()
- rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
- {'problem': rpd_file.problem.get_problems(),
- 'file': rpd_file.full_file_name}
-
- logger.error("Failed to download file: %s", rpd_file.full_file_name)
- self.update_progress(rpd_file.size, rpd_file.size)
-
- # increment this amount regardless of whether the copy actually
- # succeeded or not. It's neccessary to keep the user informed.
- self.total_downloaded += rpd_file.size
-
- try:
- copy_file_metadata(rpd_file.full_file_name, temp_full_file_name, logger)
- except:
- logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.full_file_name)
-
- # copy THM (video thumbnail file) if there is one
- if copy_succeeded and rpd_file.thm_full_name:
- # reuse video's file name
- temp_thm_full_name = temp_full_file_name + '__rpd__thm'
- try:
- shutil.copyfile(rpd_file.thm_full_name, temp_thm_full_name)
- rpd_file.temp_thm_full_name = temp_thm_full_name
- logger.debug("Copied video THM file %s", rpd_file.temp_thm_full_name)
- except (IOError, OSError) as inst:
- logger.error("Failed to download video THM file: %s", rpd_file.thm_full_name)
- logger.error("%s: %s", inst.errno, inst.strerror)
- try:
- copy_file_metadata(rpd_file.thm_full_name, temp_thm_full_name. logger)
- except:
- logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.thm_full_name)
-
- else:
- temp_thm_full_name = None
-
- #copy audio file if there is one
- if copy_succeeded and rpd_file.audio_file_full_name:
- # reuse photo's file name
- temp_audio_full_name = temp_full_file_name + '__rpd__audio'
- try:
- shutil.copyfile(rpd_file.audio_file_full_name, temp_audio_full_name)
- rpd_file.temp_audio_full_name = temp_audio_full_name
- logger.debug("Copied audio file %s", rpd_file.temp_audio_full_name)
- except (IOError, OSError) as inst:
- logger.error("Failed to download audio file: %s", rpd_file.audio_file_full_name)
- logger.error("%s: %s", inst.errno, inst.strerror)
- try:
- copy_file_metadata(rpd_file.audio_file_full_name, temp_audio_full_name, logger)
- except:
- logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.audio_file_full_name)
-
-
-
- if copy_succeeded and rpd_file.generate_thumbnail:
- thumbnail, thumbnail_icon = self.thumbnail_maker.get_thumbnail(
- temp_full_file_name,
- temp_thm_full_name,
- rpd_file.file_type,
- (160, 120), (100,100))
- else:
- thumbnail = None
- thumbnail_icon = None
-
- if copy_succeeded and self.verify_file:
- rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()
-
- if rpd_file.metadata is not None:
- rpd_file.metadata = None
-
-
- download_count = i + 1
- if self.modify_files_during_download and copy_succeeded:
- copy_finished = download_count == self.no_files
-
- self.modify_pipe.send((rpd_file, download_count, temp_full_file_name,
- thumbnail_icon, thumbnail, copy_finished))
- else:
- self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_FILE,
- (copy_succeeded, rpd_file, download_count,
- temp_full_file_name,
- thumbnail_icon, thumbnail))))
-
-
- self.results_pipe.send((rpdmp.CONN_COMPLETE, None))
-
-
- def _get_dest_dir(self, file_type):
- if file_type == rpdfile.FILE_TYPE_PHOTO:
- return self.photo_temp_dir
- else:
- return self.video_temp_dir
-
- def _create_temp_dir(self, folder):
- try:
- temp_dir = tempfile.mkdtemp(prefix="rpd-tmp-", dir=folder)
- except OSError, (errno, strerror):
- # FIXME: error reporting
- logger.error("Failed to create temporary directory in %s: %s %s",
- errono,
- strerror,
- folder)
- temp_dir = None
-
- return temp_dir
-
- def create_temp_dirs(self):
- self.photo_temp_dir = self.video_temp_dir = None
- if self.photo_download_folder is not None:
- self.photo_temp_dir = self._create_temp_dir(self.photo_download_folder)
- logger.debug("Photo temporary directory: %s", self.photo_temp_dir)
- if self.video_download_folder is not None:
- self.video_temp_dir = self._create_temp_dir(self.video_download_folder)
- logger.debug("Video temporary directory: %s", self.video_temp_dir)
-
-
-