path: root/raphodo/thumbnailpara.py

#!/usr/bin/env python3

# Copyright (C) 2011-2017 Damon Lynch <damonlynch@gmail.com>

# This file is part of Rapid Photo Downloader.
#
# Rapid Photo Downloader is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Rapid Photo Downloader is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Rapid Photo Downloader.  If not,
# see <http://www.gnu.org/licenses/>.

"""
Worker process to get thumbnails from Thumbnail or FDO cache, or
read thumbnail / file from the device being downloaded from.

For each device, there is one of these workers.

Sends thumbnail processing tasks to load balancer, which will in turn
send it to extractors.

By default, will set extractors to get the file's metadata time if
the metadata time is not already found in the rpd_file.
"""

__author__ = 'Damon Lynch'
__copyright__ = "Copyright 2011-2017, Damon Lynch"

try:
    using_injected = 'profile' in dict(__builtins__)
except:
    using_injected = False
finally:
    if not using_injected:
        # use of line_profiler not detected
        def profile(func):
            def inner(*args, **kwargs):
                return func(*args, **kwargs)
            return inner

import os
import sys
import logging
import pickle
from collections import deque
from operator import attrgetter
from typing import Optional, Tuple, Set

import zmq
from PyQt5.QtGui import QImage
from PyQt5.QtCore import QSize
import psutil
import gphoto2 as gp
try:
    import rawkit
    have_rawkit = True
except ImportError:
    have_rawkit = False

from raphodo.rpdfile import RPDFile
from raphodo.interprocess import (WorkerInPublishPullPipeline,
                          GenerateThumbnailsArguments,
                          GenerateThumbnailsResults,
                          ThumbnailExtractorArgument)
from raphodo.constants import (FileType, ThumbnailSize, ThumbnailCacheStatus,
                       ThumbnailCacheDiskStatus, ExtractionTask,
                       ExtractionProcessing, orientation_offset, thumbnail_offset,
                       ThumbnailCacheOrigin, datetime_offset)
from raphodo.camera import Camera, CameraProblemEx
from raphodo.cache import ThumbnailCacheSql, FdoCacheLarge
from raphodo.utilities import (GenerateRandomFileName, create_temp_dir, CacheDirs)
from raphodo.preferences import Preferences
from raphodo.rescan import RescanCamera


def split_list(alist: list, wanted_parts=2):
    """
    Split list into smaller parts
    http://stackoverflow.com/questions/752308/split-list-into-smaller-lists
    :param alist: the list
    :param wanted_parts: how many lists it should be split into
    :return: the split lists
    """
    length = len(alist)
    return [alist[i * length // wanted_parts: (i + 1) * length // wanted_parts]
            for i in range(wanted_parts)]


def split_indexes(length: int):
    """
    For the length of a list, return a list of indexes into it such
    that the indexes start with the middle item, then the middle item
    of the remaining two parts of the list, and so forth.

    Perhaps this algorithm could be optimized, as I did it myself. But
    hey it works and for now that's the main thing.

    :param length: the length of the list i.e. the number of indexes
     to be created
    :return: the list of indexes
    """
    l = list(range(length))
    n = []
    master = deque([l])
    while master:
        l1, l2 = split_list(master.popleft())
        if l2:
            n.append(l2[0])
            l2 = l2[1:]
        if l1:
            master.append(l1)
        if l2:
            master.append(l2)
    return n


def get_temporal_gaps_and_sequences(rpd_files, temporal_span):
    """
    For a sorted list of rpd_files, identify those rpd_files which are
    more than the temporal span away from each other, and those which are
    less than the temporal span from each other.

    Does not analyze clusters.

    For instance, you have 1000 photos from a day's photography. You
    sort them into a list ordered by time, earliest to latest. You then
    get all the photos that were take more than an hour after the
    previous photo, and those that were taken within an hour of the
    previous photo.
    .
    :param rpd_files: the sorted list of rpd_files, earliest first
    :param temporal_span: the time span that triggers a gap
    :return: the rpd_files that signify gaps, and all the rest of the
    rpd_files (which are in sequence)
    """
    if rpd_files:
        prev = rpd_files[0]
        gaps = [prev]
        sequences = []
        for i, rpd_file in enumerate(rpd_files[1:]):
            if rpd_file.modification_time - prev.modification_time > \
                    temporal_span:
                gaps.append(rpd_file)
            else:
                sequences.append(rpd_file)
            prev = rpd_file
        return (gaps, sequences)
    return None


class GetThumbnailFromCache:
    """
    Try to get thumbnail from Rapid Photo Downloader's thumbnail cache
    or from the FreeDesktop.org cache.
    """

    def __init__(self, use_thumbnail_cache: bool) -> None:

        if use_thumbnail_cache:
            self.thumbnail_cache = ThumbnailCacheSql(create_table_if_not_exists=False)
        else:
            self.thumbnail_cache = None

        # Access large size Freedesktop.org thumbnail cache
        self.fdo_cache_large =  FdoCacheLarge()

        self.thumbnail_size_needed = QSize(ThumbnailSize.width, ThumbnailSize.height)

    def image_large_enough(self, size: QSize) -> bool:
        """Check if image is equal or bigger than thumbnail size."""
        return (size.width() >= self.thumbnail_size_needed.width() or
                size.height() >= self.thumbnail_size_needed.height())

    def get_from_cache(self, rpd_file: RPDFile,
                       use_thumbnail_cache: bool=True
                       ) -> Tuple[ExtractionTask, bytes, str, ThumbnailCacheOrigin]:
        """
        Attempt to get a thumbnail for the file from the Rapid Photo Downloader thumbnail cache
        or from the FreeDesktop.org 256x256 thumbnail cache.

        :param rpd_file:
        :param use_thumbnail_cache: whether to use the
        :return:
        """

        task = ExtractionTask.undetermined
        thumbnail_bytes = None
        full_file_name_to_work_on = ''
        origin = None  # type: Optional[ThumbnailCacheOrigin]

        # Attempt to get thumbnail from Thumbnail Cache
        # (see cache.py for definitions of various caches)
        if self.thumbnail_cache is not None and use_thumbnail_cache:
            get_thumbnail = self.thumbnail_cache.get_thumbnail_path(
                full_file_name=rpd_file.full_file_name,
                mtime=rpd_file.modification_time,
                size=rpd_file.size,
                camera_model=rpd_file.camera_model)
            rpd_file.thumbnail_cache_status = get_thumbnail.disk_status
            if get_thumbnail.disk_status != ThumbnailCacheDiskStatus.not_found:
                origin = ThumbnailCacheOrigin.thumbnail_cache
                task = ExtractionTask.bypass
                if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.failure:
                    rpd_file.thumbnail_status = ThumbnailCacheStatus.generation_failed
                    rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.failure
                elif get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found:
                    rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.found
                    if get_thumbnail.orientation_unknown:
                        rpd_file.thumbnail_status = ThumbnailCacheStatus.orientation_unknown
                    else:
                        rpd_file.thumbnail_status = ThumbnailCacheStatus.ready
                    with open(get_thumbnail.path, 'rb') as thumbnail:
                        thumbnail_bytes = thumbnail.read()

        # Attempt to get thumbnail from large FDO Cache if not found in Thumbnail Cache
        # and it's not being downloaded directly from a camera (if it's from a camera, it's
        # not going to be in the FDO cache)

        if task == ExtractionTask.undetermined and not rpd_file.from_camera:
            get_thumbnail = self.fdo_cache_large.get_thumbnail(
                full_file_name=rpd_file.full_file_name,
                modification_time=rpd_file.modification_time,
                size=rpd_file.size,
                camera_model=rpd_file.camera_model)
            if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found:
                rpd_file.fdo_thumbnail_256_name = get_thumbnail.path
                thumb = get_thumbnail.thumbnail  # type: QImage
                if thumb is not None:
                    if self.image_large_enough(thumb.size()):
                        task = ExtractionTask.load_file_directly
                        full_file_name_to_work_on = get_thumbnail.path
                        origin = ThumbnailCacheOrigin.fdo_cache
                        rpd_file.thumbnail_status = ThumbnailCacheStatus.fdo_256_ready

        return task, thumbnail_bytes, full_file_name_to_work_on, origin

# How much of the file should be read in from local disk and thus cached
# by they kernel
cached_read = dict(
    cr2=260 * 1024,
    dng=504 * 1024,
    nef=400* 1024
)


def preprocess_thumbnail_from_disk(rpd_file: RPDFile,
                                   processing: Set[ExtractionProcessing]) -> ExtractionTask:
    """
    Determine how to get a thumbnail from a photo or video that is not on a camera
    (although it may have directly come from there during the download process)

    Does not return the name of the file to be worked on -- that's the responsibility
    of the method calling it.

    :param rpd_file: details about file from which to get thumbnail from
    :param processing: set that holds processing tasks for the extractors to perform
    :return: extraction task required
    """

    if rpd_file.file_type == FileType.photo:
        if rpd_file.is_tiff():
            available = psutil.virtual_memory().available
            if rpd_file.size <= available:
                bytes_to_read = rpd_file.size
                if rpd_file.mdatatime:
                    task = ExtractionTask.load_file_directly
                else:
                    task = ExtractionTask.load_file_and_exif_directly
                processing.add(ExtractionProcessing.resize)
            else:
                # Don't try to extract a thumbnail from
                # a file that is larger than available
                # memory
                task = ExtractionTask.bypass
                bytes_to_read = 0
        else:
            if rpd_file.is_jpeg() and rpd_file.from_camera and rpd_file.is_mtp_device:
                # jpeg photos from smartphones don't have embedded thumbnails
                task = ExtractionTask.load_file_and_exif_directly
                processing.add(ExtractionProcessing.resize)
            else:
                task = ExtractionTask.load_from_exif
            processing.add(ExtractionProcessing.orient)
            bytes_to_read = cached_read.get(rpd_file.extension, 400 * 1024)

        if bytes_to_read:
            if not rpd_file.download_full_file_name:
                try:
                    with open(rpd_file.full_file_name, 'rb') as photo:
                        # Bring the file into the operating system's disk cache
                        photo.read(bytes_to_read)
                except FileNotFoundError:
                    logging.error(
                        "The download file %s does not exist", rpd_file.download_full_file_name
                    )
    else:
        # video
        if rpd_file.thm_full_name is not None:
            if not rpd_file.mdatatime:
                task = ExtractionTask.load_file_directly_metadata_from_secondary
                # It's the responsibility of the calling code to assign the
                # secondary_full_file_name
            else:
                task = ExtractionTask.load_file_directly
            processing.add(ExtractionProcessing.strip_bars_video)
            processing.add(ExtractionProcessing.add_film_strip)
        else:
            if rpd_file.mdatatime:
                task = ExtractionTask.extract_from_file
            else:
                task = ExtractionTask.extract_from_file_and_load_metadata

    return task


class GenerateThumbnails(WorkerInPublishPullPipeline):

    def __init__(self) -> None:
        self.random_filename = GenerateRandomFileName()
        super().__init__('Thumbnails')

    def cache_full_size_file_from_camera(self, rpd_file: RPDFile) -> bool:
        """
        Get the file from the camera chunk by chunk and cache it.

        :return: True if operation succeeded, False otherwise
        """
        if rpd_file.file_type == FileType.photo:
            cache_dir = self.photo_cache_dir
        else:
            cache_dir = self.video_cache_dir
        cache_full_file_name = os.path.join(
            cache_dir, '{}.{}'.format(self.random_filename.name(), rpd_file.extension)
        )
        try:
            self.camera.save_file_by_chunks(
                dir_name=rpd_file.path,
                file_name=rpd_file.name,
                size=rpd_file.size,
                dest_full_filename=cache_full_file_name,
                progress_callback=None,
                check_for_command=self.check_for_controller_directive,
                return_file_bytes=False
            )
        except CameraProblemEx as e:
            #TODO report error
            return False
        else:
            rpd_file.cache_full_file_name = cache_full_file_name
            return True

    def cache_file_chunk_from_camera(self, rpd_file: RPDFile, offset: int) -> bool:
        if rpd_file.file_type == FileType.photo:
            cache_dir = self.photo_cache_dir
        else:
            cache_dir = self.video_cache_dir
        cache_full_file_name = os.path.join(
            cache_dir, '{}.{}'.format(self.random_filename.name(), rpd_file.extension)
        )
        try:
            self.camera.save_file_chunk(
                dir_name=rpd_file.path,
                file_name=rpd_file.name,
                chunk_size_in_bytes=min(offset, rpd_file.size),
                dest_full_filename=cache_full_file_name
            )
            rpd_file.temp_cache_full_file_chunk = cache_full_file_name
            return True
        except CameraProblemEx as e:
            #TODO problem reporting
            return False

    def do_work(self) -> None:
        try:
            self.generate_thumbnails()
        except SystemExit as e:
            sys.exit(e)
        except:
            if hasattr(self, 'device_name'):
                logging.error("Exception generating thumbnails for %s", self.device_name)
            else:
                logging.error("Exception generating thumbnails")
            logging.exception("Traceback:")
        
    def generate_thumbnails(self) -> None:
        self.camera = None
        arguments = pickle.loads(self.content)  # type: GenerateThumbnailsArguments
        self.device_name = arguments.name
        logging.info("Generating %s thumbnails for %s", len(arguments.rpd_files), arguments.name)
        if arguments.log_gphoto2:
            gp.use_python_logging()

        self.frontend = self.context.socket(zmq.PUSH)
        self.frontend.connect("tcp://localhost:{}".format(arguments.frontend_port))

        self.prefs = Preferences()

        # If the entire video is required to extract the thumbnail, which is determined
        # when extracting sample metadata from a video during the device scan
        entire_video_required = arguments.entire_video_required

        # Access and generate Rapid Photo Downloader thumbnail cache
        use_thumbnail_cache = self.prefs.use_thumbnail_cache

        thumbnail_caches = GetThumbnailFromCache(use_thumbnail_cache=use_thumbnail_cache)

        photo_cache_dir = video_cache_dir = None
        cache_file_from_camera = False

        rpd_files = arguments.rpd_files

        # with open('tests/thumbnail_data_medium_no_tiff', 'wb') as f:
        #     pickle.dump(rpd_files, f)

        # Must sort files by modification time prior to temporal analysis needed to figure out
        # which thumnbnails to prioritize
        rpd_files = sorted(rpd_files,  key=attrgetter('modification_time'))

        time_span = arguments.proximity_seconds

        rpd_files2 = []

        if rpd_files:
            gaps, sequences = get_temporal_gaps_and_sequences(rpd_files, time_span)

            rpd_files2.extend(gaps)

            indexes = split_indexes(len(sequences))
            rpd_files2.extend([sequences[idx] for idx in indexes])

        assert len(rpd_files) == len(rpd_files2)
        rpd_files = rpd_files2

        if arguments.camera is not None:
            self.camera = Camera(
                model=arguments.camera, port=arguments.port,
                specific_folders=self.prefs.folders_to_scan
            )

            if not self.camera.camera_initialized:
                # There is nothing to do here: exit!
                logging.debug(
                    "Prematurely exiting thumbnail generation due to lack of access to camera %s",
                    arguments.camera
                )
                self.disconnect_logging()
                self.send_finished_command()
                sys.exit(0)

            must_make_cache_dirs = (not self.camera.can_fetch_thumbnails or cache_file_from_camera)

            if must_make_cache_dirs or arguments.need_video_cache_dir:
                # If downloading complete copy of the files to
                # generate previews, then may as well cache them to speed up
                # the download process
                cache_file_from_camera = must_make_cache_dirs
                self.photo_cache_dir = create_temp_dir(
                    folder=arguments.cache_dirs.photo_cache_dir,
                    prefix='rpd-cache-{}-'.format(self.device_name[:10]))
                self.video_cache_dir = create_temp_dir(
                    folder=arguments.cache_dirs.video_cache_dir,
                    prefix='rpd-cache-{}-'.format(self.device_name[:10]))
                cache_dirs = CacheDirs(self.photo_cache_dir, self.video_cache_dir)
                self.content = pickle.dumps(GenerateThumbnailsResults(
                    scan_id=arguments.scan_id,
                    cache_dirs=cache_dirs), pickle.HIGHEST_PROTOCOL)
                self.send_message_to_sink()

        from_thumb_cache = 0
        from_fdo_cache = 0

        if self.camera:
            rescan = RescanCamera(camera=self.camera, prefs=self.prefs)
            rescan.rescan_camera(rpd_files)
            rpd_files = rescan.rpd_files
            if rescan.missing_rpd_files:
                logging.error(
                    "%s files could not be relocated on %s",
                    len(rescan.missing_rpd_files), self.camera.display_name
                )
                for rpd_file in rescan.missing_rpd_files:  # type: RPDFile
                    self.content = pickle.dumps(
                        GenerateThumbnailsResults(rpd_file=rpd_file, thumbnail_bytes=None),
                        pickle.HIGHEST_PROTOCOL
                    )
                    self.send_message_to_sink()

        for rpd_file in rpd_files:  # type: RPDFile
            # Check to see if the process has received a command
            self.check_for_controller_directive()

            exif_buffer = None
            file_to_work_on_is_temporary = False
            secondary_full_file_name = ''
            processing = set()  # type: Set[ExtractionProcessing]

            # Attempt to get thumbnail from Thumbnail Cache
            # (see cache.py for definitions of various caches)

            cache_search = thumbnail_caches.get_from_cache(rpd_file)
            task, thumbnail_bytes, full_file_name_to_work_on, origin = cache_search
            if task != ExtractionTask.undetermined:
                if origin == ThumbnailCacheOrigin.thumbnail_cache:
                    from_thumb_cache += 1
                    # logging.debug("Thumbnail for %s found in RPD thumbnail cache",
                    #               rpd_file.full_file_name)
                else:
                    assert origin == ThumbnailCacheOrigin.fdo_cache
                    logging.debug(
                        "Thumbnail for %s found in large FDO cache", rpd_file.full_file_name
                    )
                    from_fdo_cache += 1
                    processing.add(ExtractionProcessing.resize)
                    if not rpd_file.mdatatime:
                        # Since we're extracting the thumbnail from the FDO cache,
                        # need to grab its metadata too.
                        # Reassign the task
                        task = ExtractionTask.load_file_directly_metadata_from_secondary
                        # It's not being downloaded from a camera, so nothing
                        # special to do except assign the name of the file from which
                        # to extract the metadata
                        secondary_full_file_name = rpd_file.full_file_name
                        logging.debug(
                            "Although thumbnail found in the cache, tasked to extract "
                            "file time recorded in metadata from %s", secondary_full_file_name
                        )
            if task == ExtractionTask.undetermined:
                # Thumbnail was not found in any cache: extract it
                if self.camera:  # type: Camera
                    if rpd_file.file_type == FileType.photo:
                        if self.camera.can_fetch_thumbnails:
                            task = ExtractionTask.load_from_bytes
                            if rpd_file.is_jpeg_type():
                                # gPhoto2 knows how to get jpeg thumbnails
                                try:
                                    exif_buffer = self.camera.get_exif_extract_from_jpeg(
                                        rpd_file.path, rpd_file.name
                                    )
                                except CameraProblemEx as e:
                                    # TODO handle error?
                                    exif_buffer = None
                            else:
                                # gPhoto2 does not know how to get RAW thumbnails, so we do that
                                # part ourselves
                                if rpd_file.extension == 'crw':
                                    # Could cache this file, since reading its entirety
                                    # But does anyone download a CRW file from the camera these
                                    # days?!
                                    bytes_to_read = rpd_file.size
                                else:
                                    bytes_to_read = min(
                                        rpd_file.size, orientation_offset.get(
                                            rpd_file.extension, 500
                                        )
                                    )
                                exif_buffer = self.camera.get_exif_extract(
                                    rpd_file.path, rpd_file.name, bytes_to_read
                                )
                            try:
                                thumbnail_bytes = self.camera.get_thumbnail(
                                    rpd_file.path, rpd_file.name
                                )
                            except CameraProblemEx as e:
                                #TODO report error
                                thumbnail_bytes = None
                            processing.add(ExtractionProcessing.strip_bars_photo)
                            processing.add(ExtractionProcessing.orient)
                        else:
                            # Many (all?) jpegs from phones don't include jpeg previews,
                            # so need to render from the entire jpeg itself. Slow!

                            # If rawkit is not installed, then extract merely a part of
                            # phone's raw format, and try to extract the jpeg preview
                            # from it (which probably doesn't exist!). This is fast.
                            # If have rawkit, download and render an image from the
                            # RAW
                            if not rpd_file.is_jpeg() and not have_rawkit:
                                bytes_to_read = thumbnail_offset.get(rpd_file.extension)
                                if bytes_to_read:
                                    exif_buffer = self.camera.get_exif_extract(
                                        rpd_file.path, rpd_file.name, bytes_to_read
                                    )
                                    task = ExtractionTask.load_from_exif_buffer
                                    processing.add(ExtractionProcessing.orient)
                            if (task == ExtractionTask.undetermined and
                                    self.cache_full_size_file_from_camera(rpd_file)):
                                if rpd_file.is_jpeg():
                                    task = ExtractionTask.load_file_and_exif_directly
                                    processing.add(ExtractionProcessing.resize)
                                    processing.add(ExtractionProcessing.orient)
                                else:
                                    task = ExtractionTask.load_from_exif
                                    processing.add(ExtractionProcessing.resize)
                                    processing.add(ExtractionProcessing.orient)
                                full_file_name_to_work_on = rpd_file.cache_full_file_name
                            else:
                                # Failed to generate thumbnail
                                task = ExtractionTask.bypass
                    else:
                        # video from camera
                        if rpd_file.thm_full_name is not None:
                            # Fortunately, we have a special video thumbnail file
                            # Still need to get metadata time, however.

                            if entire_video_required:
                                offset = rpd_file.size
                            else:
                                offset = datetime_offset.get(rpd_file.extension)
                                # If there is no offset, there is not point trying to extract the
                                # metadata time from part of the video. It's not ideal,
                                # but if this is from a camera on which there were any other files
                                # we can assume we've got a somewhat accurate date time for it from
                                # the modification time.
                                # The only exception is if the video file is not that big, in which
                                # case it's worth reading in its entirety:
                                if offset is None and rpd_file.size < 4000000:
                                    offset = rpd_file.size

                            if rpd_file.mdatatime or not offset:
                                task = ExtractionTask.load_from_bytes
                            elif self.cache_file_chunk_from_camera(rpd_file, offset):
                                task = ExtractionTask.load_from_bytes_metadata_from_temp_extract
                                secondary_full_file_name = rpd_file.temp_cache_full_file_chunk
                            else:
                                # For some reason was unable to download part of the video file
                                task = ExtractionTask.load_from_bytes

                            try:
                                thumbnail_bytes = self.camera.get_THM_file(rpd_file.thm_full_name)
                            except CameraProblemEx as e:
                                # TODO report error
                                thumbnail_bytes = None
                            processing.add(ExtractionProcessing.strip_bars_video)
                            processing.add(ExtractionProcessing.add_film_strip)
                        else:
                            if not entire_video_required:
                                # For most videos, extract a small part of the video and use
                                # that to generate thumbnail
                                offset = thumbnail_offset.get(rpd_file.extension)
                                if offset:
                                    offset = max(offset, datetime_offset.get(rpd_file.extension))
                                if offset and self.cache_file_chunk_from_camera(rpd_file, offset):
                                    task = ExtractionTask.extract_from_file_and_load_metadata
                                    full_file_name_to_work_on = rpd_file.temp_cache_full_file_chunk
                                    file_to_work_on_is_temporary = True
                            if task == ExtractionTask.undetermined:
                                if self.cache_full_size_file_from_camera(rpd_file):
                                    task = ExtractionTask.extract_from_file_and_load_metadata
                                    full_file_name_to_work_on = rpd_file.cache_full_file_name
                                else:
                                    # Failed to generate thumbnail
                                    task = ExtractionTask.bypass
                else:
                    # File is not on a camera
                    task = preprocess_thumbnail_from_disk(rpd_file=rpd_file, processing=processing)
                    if task != ExtractionTask.bypass:
                        if rpd_file.thm_full_name is not None:
                            full_file_name_to_work_on = rpd_file.thm_full_name
                            if task == ExtractionTask.load_file_directly_metadata_from_secondary:
                                secondary_full_file_name = rpd_file.full_file_name
                        else:
                            full_file_name_to_work_on = rpd_file.full_file_name

            if task == ExtractionTask.bypass:
                self.content = pickle.dumps(
                    GenerateThumbnailsResults(rpd_file=rpd_file, thumbnail_bytes=thumbnail_bytes),
                    pickle.HIGHEST_PROTOCOL
                )
                self.send_message_to_sink()

            elif task != ExtractionTask.undetermined:
                # Send data to load balancer, which will send to one of its
                # workers

                self.content = pickle.dumps(
                    ThumbnailExtractorArgument(
                        rpd_file=rpd_file,
                        task=task,
                        processing=processing,
                        full_file_name_to_work_on=full_file_name_to_work_on,
                        secondary_full_file_name=secondary_full_file_name,
                        exif_buffer=exif_buffer,
                        thumbnail_bytes = thumbnail_bytes,
                        use_thumbnail_cache=use_thumbnail_cache,
                        file_to_work_on_is_temporary=file_to_work_on_is_temporary,
                        write_fdo_thumbnail=False,
                        send_thumb_to_main=True
                    ),
                    pickle.HIGHEST_PROTOCOL)
                self.frontend.send_multipart([b'data', self.content])

        if arguments.camera:
            self.camera.free_camera()
            # Delete our temporary cache directories if they are empty
            if photo_cache_dir is not None:
                if not os.listdir(self.photo_cache_dir):
                    os.rmdir(self.photo_cache_dir)
            if video_cache_dir is not None:
                if not os.listdir(self.video_cache_dir):
                    os.rmdir(self.video_cache_dir)

        logging.debug("Finished phase 1 of thumbnail generation for %s", self.device_name)
        if from_thumb_cache:
            logging.info(
                "{} of {} thumbnails for {} came from thumbnail cache".format(
                    from_thumb_cache, len(rpd_files), self.device_name
                )
            )
        if from_fdo_cache:
            logging.info(
                "{} of {} thumbnails of for {} came from Free Desktop cache".format(
                    from_fdo_cache, len(rpd_files), self.device_name
                )
            )

        self.disconnect_logging()
        self.send_finished_command()

    def cleanup_pre_stop(self):
        if self.camera is not None:
            self.camera.free_camera()

if __name__ == "__main__":
    generate_thumbnails = GenerateThumbnails()