import os
import random
import re
import threading
from glob import glob

from .collections import StatefulCollection
from .exceptions import (NoAvailableWorkException, SourceNotNamedException)
from .. import events
from .. import logger
from .. import tasks
from ..progress import FileLockedException
from ...astrom import StreamingAstromWriter, Source, SourceReading, StreamingVettingWriter
from ...orbfit import Orbfit

__author__ = "David Rusk <drusk@uvic.ca>"

class WorkUnit(object):
    """
    A unit of work to be processed, associated with the data from a single
    input file.
    """

    def __init__(self, filename,
                 parsed_data,
                 progress_manager,
                 output_context,
                 dry_run=False):
        self.filename = filename
        self.data = parsed_data
        self.progress_manager = progress_manager
        self.output_context = output_context
        self.dry_run = dry_run

        self.sources = StatefulCollection(parsed_data.get_sources())

        self.readings_by_source = {}
        for source in self.sources:
            self.readings_by_source[source] = StatefulCollection(source.get_readings())

        self.processed_items = set()
        self._mark_previously_processed_items()

        if self.is_current_item_processed():
            self.next_item()

        self.finished_callbacks = []

        self._unlocked = False

    def register_finished_callback(self, callback):
        self.finished_callbacks.append(callback)

    def next_source(self):
        next(self.get_sources())

    def previous_source(self):
        self.get_sources().previous()

    def next_obs(self):
        next(self.get_current_source_readings())

    def previous_obs(self):
        self.get_current_source_readings().previous()

    def next_item(self):
        raise NotImplementedError()

    def accept_current_item(self):
        self.process_current_item()

    def reject_current_item(self):
        self.process_current_item()

    def process_current_item(self):
        self.processed_items.add(self.get_current_item())

        if self.is_current_source_finished():
            self.progress_manager.record_index(self.get_filename(),
                                               self.get_current_source_number())
            self.get_writer().flush()

        if self.is_finished():
            self.progress_manager.record_done(self.get_filename())
            self.unlock()

            for callback in self.finished_callbacks:
                callback(self.get_results_file_paths())

            self._close_writers()

    def get_current_item(self):
        raise NotImplementedError()

    def get_current_item_index(self):
        raise NotImplementedError()

    def is_item_processed(self, item):
        return item in self.processed_items

    def is_current_item_processed(self):
        return self.is_item_processed(self.get_current_item())

    def is_source_finished(self, source):
        raise NotImplementedError()

    def is_current_source_finished(self):
        return self.is_source_finished(self.get_current_source())

    def get_filename(self):
        return self.filename

    def get_data(self):
        return self.data

    def get_sources(self):
        return self.sources

    def get_unprocessed_sources(self):
        unprocessed_sources = []
        for source in self.get_sources():
            if not self.is_source_finished(source):
                unprocessed_sources.append(source)

        return unprocessed_sources

    def get_source_count(self):
        return len(self.get_sources())

    def get_current_source(self):
        source = self.get_sources().get_current_item()
        assert isinstance(source, Source)
        return source

    def get_current_source_number(self):
        return self.get_sources().get_index()

    def get_current_source_readings(self):
        return self.readings_by_source[self.get_current_source()]

    def get_obs_count(self):
        return len(self.get_current_source_readings())

    def get_current_reading(self):
        """
        :return: SourceReading
        :rtype: SourceReading
        """
        return self.get_current_source_readings().get_current_item()

    def get_current_obs_number(self):
        return self.get_current_source_readings().get_index()

    def get_writer(self):
        raise NotImplementedError()

    def get_results_file_paths(self):
        raise NotImplementedError()

    def is_finished(self):
        return len(self._get_item_set() - self.processed_items) == 0

    def is_apcor_needed(self):
        raise NotImplementedError()

    def unlock(self):
        if not self._unlocked:
            self.progress_manager.unlock(self.get_filename(), do_async=True)
            self._unlocked = True

    def _get_item_set(self):
        raise NotImplementedError()

    def _mark_previously_processed_items(self):
        pass

    def _close_writers(self):
        pass

class RealsWorkUnit(WorkUnit):
    """
    A unit of work when performing the process reals task.
    """

    def __init__(self,
                 filename,
                 parsed_data,
                 progress_manager,
                 output_context,
                 dry_run=False):
        super(RealsWorkUnit, self).__init__(
            filename,
            parsed_data,
            progress_manager,
            output_context,
            dry_run=dry_run)
        self._writers = {}

    def next_item(self):
        assert not self.is_finished()

        self.next_obs()
        while self.is_current_item_processed():
            self._next_sequential_item()

    def _next_sequential_item(self):
        """
        Go to the next item in the 'ideal' processing sequence.
        """
        if self.get_current_source_readings().is_on_last_item():
            self.next_source()
        else:
            self.next_obs()

    def get_current_item(self):
        return self.get_current_reading()

    def get_current_item_index(self):
        return (self.get_sources().get_index() * self.get_obs_count() +
                self.get_current_source_readings().get_index())

    def is_source_finished(self, source):
        for reading in source.get_readings():
            if reading not in self.processed_items:
                return False

        return True

    def is_apcor_needed(self):
        return True

    def get_writer(self):
        filename = self.get_output_filename(self.get_current_source())
        if filename in self._writers:
            return self._writers[filename]

        writer = self._create_writer(filename)
        self._writers[filename] = writer
        return writer

    def get_results_file_paths(self):
        return [self.output_context.get_full_path(filename)
                for filename in self._writers]

    def get_output_filename(self, source):
        if not source.has_provisional_name():
            raise SourceNotNamedException(source)

        return ".".join((os.path.basename(self.filename),
                         source.get_provisional_name(),
                         "mpc"))

    def _create_writer(self, filename):
        # NOTE: this import is only here so that we don't load up secondary
        # dependencies (like astropy) used in MPCWriter when they are not
        # needed (i.e. cands task).  This is to help reduce the application
        # startup time.
        from ...mpc import MPCWriter

        return MPCWriter(self.output_context.open(filename),
                         auto_flush=False)

    def _get_item_set(self):
        all_readings = set()
        for readings in self.readings_by_source.values():
            all_readings.update(readings)
        return all_readings

    def _mark_previously_processed_items(self):
        processed_indices = self.progress_manager.get_processed_indices(self.get_filename())
        for index in processed_indices:
            for reading in self.get_sources()[index].get_readings():
                self.processed_items.add(reading)

    def _close_writers(self):
        for writer in list(self._writers.values()):
            writer.close()

class CandidatesWorkUnit(WorkUnit):
    """
    A unit of work when performing the process candidates task.
    """

    def __init__(self,
                 filename,
                 parsed_data,
                 progress_manager,
                 output_context,
                 dry_run=False):
        super(CandidatesWorkUnit, self).__init__(
            filename,
            parsed_data,
            progress_manager,
            output_context,
            dry_run=dry_run)

        self._writer = None

    def next_item(self):
        assert not self.is_finished()

        self.next_source()
        while self.is_current_item_processed():
            self.next_source()

    def get_current_item(self):
        return self.get_current_source()

    def get_current_item_index(self):
        return self.get_sources().get_index()

    def is_source_finished(self, source):
        return source in self.processed_items

    def is_apcor_needed(self):
        return False

    def get_writer(self):
        if self._writer is None:
            self._writer = self._create_writer()

        return self._writer

    def get_results_file_paths(self):
        if self._writer is None:
            return []

        return [self.output_context.get_full_path(self._writer.get_filename())]

    def get_output_filename(self):
        return self.get_filename().replace(tasks.get_suffix(tasks.CANDS_TASK),
                                           tasks.get_suffix(tasks.REALS_TASK))

    def _create_writer(self):
        filename = self.get_output_filename()
        return StreamingAstromWriter(self.output_context.open(filename),
                                     self.data.sys_header)

    def _get_item_set(self):
        return set(self.sources)

    def _mark_previously_processed_items(self):
        processed_indices = self.progress_manager.get_processed_indices(self.get_filename())
        for index in processed_indices:
            self.processed_items.add(self.get_sources()[index])

    def _close_writers(self):
        if self._writer is not None:
            self._writer.close()

class VettingWorkUnit(CandidatesWorkUnit):
    """
    A unit of work when performing the process candidates task.
    """

    def _create_writer(self):
        filename = self.get_output_filename()
        return StreamingVettingWriter(self.output_context.open(filename))

    def get_output_filename(self):

        suffix = tasks.get_suffix(tasks.VETTING_TASK)
        try:
            base_name = re.search("(?P<base_name>.*?)\.\d*{}".format(suffix), self.filename).group('base_name')
        except:
            base_name = os.path.splitext(self.filename)[0]
        filename_pattern = self.output_context.get_full_path(
            "{}.?{}".format(base_name, suffix))
        file_count = len(glob(filename_pattern))
        filename = self.output_context.get_full_path(
            "{}.{}{}".format(base_name, file_count, suffix))
        return filename

class TracksWorkUnit(WorkUnit):
    """
    A unit of work when performing the process track task.
    """

    def __init__(self,
                 builder,
                 filename,
                 parsed_data,
                 progress_manager,
                 output_context,
                 dry_run=False):
        super(TracksWorkUnit, self).__init__(
            filename,
            parsed_data,
            progress_manager,
            output_context,
            dry_run=dry_run)

        self.builder = builder
        self._writer = None
        self._ssos_queried = False
        self._comparitors = {}

    def print_orbfit_info(self):
        # TODO: this should not be here.
        orb = Orbfit(self.get_writer().get_chronological_buffered_observations())
        print(orb.residuals)
        print(orb)

    def query_ssos(self):
        """
        Use the MPC file that has been built up in processing this work
        unit to generate another workunit.
        """
        self._ssos_queried = True
        mpc_filename = self.save()
        return self.builder.build_workunit(mpc_filename)

    def save(self):
        """
        Update the SouceReading information for the currently recorded observations and then flush those to a file.
        @return: mpc_filename of the resulting save.
        """
        self.get_writer().flush()
        mpc_filename = self.get_writer().get_filename()
        self.get_writer().close()
        self._writer = None
        return mpc_filename

    def is_finished(self):
        return self._ssos_queried or self.get_source_count() == 0 or super(TracksWorkUnit, self).is_finished()

    def next_item(self):
        assert not self.is_finished()

        self.next_obs()
        while self.is_current_item_processed():
            self._next_sequential_item()

    def _next_sequential_item(self):
        """
        Go to the next item to process.
        """
        if self.get_current_source_readings().is_on_last_item():
            self.next_source()
        else:
            self.next_obs()

    def get_current_item(self):
        return self.get_current_reading()

    def get_current_item_index(self):
        return (self.get_sources().get_index() * self.get_obs_count() +
                self.get_current_source_readings().get_index())

    def is_source_finished(self, source):
        for reading in source.get_readings():
            if reading not in self.processed_items:
                return False

        return True

    def is_apcor_needed(self):
        return True

    def get_writer(self):
        """
        Get a writer.

        This method also makes the output filename be the same as the .track file but with .mpc.
        (Currently only works on local filesystem)
        :rtype MPCWriter
        """
        if self._writer is None:
            suffix = tasks.get_suffix(tasks.TRACK_TASK)
            try:
                base_name = re.search("(?P<base_name>.*?)\.\d*{}".format(suffix), self.filename).group('base_name')
            except:
                base_name = os.path.splitext(self.filename)[0]
            mpc_filename_pattern = self.output_context.get_full_path(
                "{}.?{}".format(base_name, suffix))
            mpc_file_count = len(glob(mpc_filename_pattern))
            mpc_filename = "{}.{}{}".format(base_name, mpc_file_count, suffix)
            self._writer = self._create_writer(mpc_filename)

        return self._writer

    def get_results_file_paths(self):
        if self._writer is None:
            return []

        return [self.output_context.get_full_path(self._writer.get_filename())]

    def _create_writer(self, filename):
        # NOTE: this import is only here so that we don't load up secondary
        # dependencies (like astropy) used in MPCWriter when they are not
        # needed (i.e. cands task).  This is to help reduce the application
        # startup time.
        from ...mpc import MPCWriter

        writer = MPCWriter(self.output_context.open(filename),
                           auto_flush=False, auto_discovery=False)

        for rawname in self.data.mpc_observations:
            writer.write(self.data.mpc_observations[rawname])

        return writer

    def _get_item_set(self):
        all_readings = set()
        for readings in self.readings_by_source.values():
            all_readings.update(readings)
        return all_readings

    def _mark_previously_processed_items(self):
        processed_indices = self.progress_manager.get_processed_indices(self.get_filename())
        for index in processed_indices:
            for reading in self.get_sources()[index].get_readings():
                self.processed_items.add(reading)

    def _close_writers(self):
        if self._writer is not None:
            self._writer.close()

class TargetWorkUnit(TracksWorkUnit):

    def __init__(self,
                 builder,
                 filename,
                 parsed_data,
                 progress_manager,
                 output_context,
                 dry_run=False):
        super(TracksWorkUnit, self).__init__(
            filename,
            parsed_data,
            progress_manager,
            output_context,
            dry_run=dry_run)

class WorkUnitProvider(object):
    """
    Obtains new units of work for the application.
    """

    def __init__(self,
                 taskid,
                 directory_context,
                 progress_manager,
                 builder,
                 randomize=False,
                 name_filter=None):
        self.taskid = taskid
        self.directory_context = directory_context
        self.progress_manager = progress_manager
        self.builder = builder
        self.randomize = randomize
        self.name_filter = name_filter
        self._done = []
        self._already_fetched = []

    @property
    def directory(self):
        """
        The directory that workunits are being acquired from.
        """
        return self.directory_context.directory

    def _filter(self, filename):
        """
        return 'true' if filename doesn't match name_filter regex and should be filtered out of the list.
        @param filename:
        @return:
        """
        return self.name_filter is not None and re.search(self.name_filter, filename) is None

    def get_workunit(self, ignore_list=None):
        """
        Gets a new unit of work.

        Args:
          ignore_list: list(str)
            A list of filenames which should be ignored.  Defaults to None.

        Returns:
          new_workunit: WorkUnit
            A new unit of work that has not yet been processed.  A lock on
            it has been acquired.

        Raises:
          NoAvailableWorkException
            There is no more work available.
        """
        if ignore_list is None:
            ignore_list = []

        potential_files = self.get_potential_files(ignore_list)

        while len(potential_files) > 0:
            potential_file = self.select_potential_file(potential_files)
            potential_files.remove(potential_file)
            if self._filter(potential_file):
                continue

            if self.directory_context.get_file_size(potential_file) == 0:
                continue

            if self.progress_manager.is_done(potential_file):
                self._done.append(potential_file)
                continue
            else:
                try:
                    self.progress_manager.lock(potential_file)
                except FileLockedException:
                    continue

                self._already_fetched.append(potential_file)
                return self.builder.build_workunit(
                    self.directory_context.get_full_path(potential_file))

        logger.info("No eligible workunits remain to be fetched.")

        raise NoAvailableWorkException()

    def get_potential_files(self, ignore_list):
        """
        Get a listing of files for the appropriate task which may or may
        not be locked and/or done.
        """
        logger.debug(f"Getting a list of potential files using {self.directory_context}")
        exclude_prefix = self.taskid == tasks.suffixes.get(tasks.REALS_TASK, '') and 'fk' or None
        filenames = [filename for filename in
                     self.directory_context.get_listing(self.taskid, exclude_prefix=exclude_prefix)
                     if filename not in ignore_list and
                     filename not in self._done and
                     filename not in self._already_fetched]

        # if the extension is .mpc. then we look for the largest numbered MPC file.
        # look for the largest numbered MPC file only.
        if self.taskid == tasks.suffixes.get(tasks.TRACK_TASK, ''):
            basenames = {}
            for filename in filenames:
                fullname = os.path.splitext(filename)[0]
                if fullname in basenames:
                    continue
                basename = os.path.splitext(fullname)[0]
                # only do the 'maximum' search when the 2nd extension is an integer value
                try:
                    idx = int(filename.split('.')[-2])
                    if idx > basenames.get(basename, 0):
                        basenames[basename] = idx
                except:
                    # since we failed, just keep the file in the list
                    basenames[fullname] = ''
            filenames = []
            for basename in basenames:
                # sometimes the version is empty, so no '.' is needed
                version = basenames[basename]
                version = len(str(version)) > 0 and ".{}".format(version) or version
                filenames.append("{}{}{}".format(basename, version, self.taskid))
                # print basename, basenames[basename], filenames[-1]
        logger.debug(f"Got a list of these files {filenames}")

        return filenames

    def select_potential_file(self, potential_files):
        if self.randomize:
            # Don't want predictable patterns in the order we get work units.
            return random.choice(potential_files)
        else:
            return potential_files[0]

    def shutdown(self):
        pass

class PreFetchingWorkUnitProvider(object):
    def __init__(self, workunit_provider, prefetch_quantity, image_manager):
        self.workunit_provider = workunit_provider
        self.prefetch_quantity = prefetch_quantity
        self.image_manager = image_manager

        self.fetched_files = []
        self.workunits = []

        self._threads = []
        self._all_fetched = False

    @property
    def directory(self):
        """
        The directory that workunits are being acquired from.
        """
        return self.workunit_provider.directory

    def get_workunit(self):
        if self._all_fetched and len(self.workunits) == 0:
            raise NoAvailableWorkException()

        if len(self.workunits) > 0:
            workunit = self.workunits.pop(0)
        else:
            logger.debug(f"No workunits available, getting some using {self.workunit_provider}")
            workunit = self.workunit_provider.get_workunit(
                ignore_list=self.fetched_files)
            self.fetched_files.append(workunit.get_filename())

        self.trigger_prefetching()
        logger.debug("Returning {}".format(workunit))
        return workunit

    def trigger_prefetching(self):
        if self._all_fetched:
            return

        num_to_fetch = self.prefetch_quantity - len(self.workunits)
        logger.info(f"Fetching {num_to_fetch} files.")
        if num_to_fetch < 0:
            # Prefetch quantity can be 0
            num_to_fetch = 0

        while num_to_fetch > 0:
            if self._all_fetched:
                return

            self.prefetch_workunit()
            num_to_fetch -= 1

    def prefetch_workunit(self):
        thread = threading.Thread(target=self._do_prefetch_workunit)
        self._threads.append(thread)
        logger.debug(f"Starting prefetch thread.")
        thread.start()
        logger.debug(f"Prefetch started.")

    def _do_prefetch_workunit(self):
        try:
            workunit = self.workunit_provider.get_workunit(
                ignore_list=self.fetched_files)
            filename = workunit.get_filename()

            # 2 or more threads created back to back could end up
            # retrieving the same workunit.  Only keep one of them.
            if filename not in self.fetched_files:
                self.fetched_files.append(filename)
                self.workunits.append(workunit)
                self.image_manager.download_singlets_for_workunit(workunit)

                logger.info("%s was prefetched." % filename)

        except NoAvailableWorkException:
            self._all_fetched = True

    def shutdown(self):
        # Make sure all threads are finished so that no more locks are
        # acquired
        for thread in self._threads:
            thread.join()

        for workunit in self.workunits:
            workunit.unlock()

class WorkUnitBuilder(object):
    """
    Used to construct a WorkUnit with its necessary components.
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        logger.debug("Setting up Work Unit Builder.")
        self.parser = parser
        self.input_context = input_context
        self.output_context = output_context
        self.progress_manager = progress_manager
        self.dry_run = dry_run

    def build_workunit(self, input_fullpath):
        try:
            parsed_data = self.parser.parse(input_fullpath)
            logger.debug("Parsed %s (%d sources)" %
                         (input_fullpath, parsed_data.get_source_count()))

            input_filename = os.path.split(input_fullpath)[1]
            return self._do_build_workunit(
                input_filename,
                parsed_data,
                self.progress_manager,
                self.output_context,
                self.dry_run)

        except AssertionError as e:
            logger.critical(str(e))
            events.send(events.NO_AVAILABLE_WORK)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):
        raise NotImplementedError()

class TracksWorkUnitBuilder(WorkUnitBuilder):
    """
    Used to construct a WorkUnit for doing 'Track'
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(TracksWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run
        )

    @staticmethod
    def get_readings(data):
        # Note: Track workunits only have 1 source
        return data.get_sources()[0].get_readings()

    @staticmethod
    def set_readings(data, readings):
        # Note: Track workunits only have 1 source
        data.get_sources()[0].readings = readings

    def get_discovery_index(self, data):
        # Note: there should only be one reading marked "discovery" amongst
        # the data.
        for i, reading in enumerate(self.get_readings(data)):
            if reading.discovery:
                return i
        logger.warning("No discovery index found in track workunit.")
        return None

    def move_discovery_to_front(self, data):
        """
        Moves the discovery triplet to the front of the reading list.
        Leaves everything else in the same order.
        """
        readings = self.get_readings(data)
        discovery_index = self.get_discovery_index(data)

        reordered_readings = (readings[discovery_index:discovery_index + 3] +
                              readings[:discovery_index] +
                              readings[discovery_index + 3:])

        self.set_readings(data, reordered_readings)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):

        # Doesn't work... TODO: FixMe
        # self.move_discovery_to_front(data)

        return TracksWorkUnit(self, filename, data, progress_manager, output_context, dry_run=dry_run)

class TargetWorkUnitBuilder(TracksWorkUnitBuilder):

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(TracksWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run)

    @staticmethod
    def get_readings(data):
        # Note: Track workunits only have 1 source
        return data.get_sources()[0].get_readings()

    @staticmethod
    def set_readings(data, readings):
        # Note: Track workunits only have 1 source
        data.get_sources()[0].readings = readings

    def get_discovery_index(self, data):
        return None

    def move_discovery_to_front(self, data):
        pass

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):

        # Doesn't work... TODO: FixMe
        # self.move_discovery_to_front(data)

        return TracksWorkUnit(self, filename, data, progress_manager, output_context, dry_run=dry_run)

class RealsWorkUnitBuilder(WorkUnitBuilder):
    """
    Used to construct a WorkUnit with its necessary components.
    Constructs RealsWorkUnits for the process reals task.
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(RealsWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):
        return RealsWorkUnit(
            filename, data, progress_manager, output_context,
            dry_run=dry_run)

class CandidatesWorkUnitBuilder(WorkUnitBuilder):
    """
    Used to construct a WorkUnit with its necessary components.
    Constructs CandidatesWorkUnits for the process candidates task.
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(CandidatesWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):
        return CandidatesWorkUnit(
            filename, data, progress_manager, output_context,
            dry_run=dry_run)

class VettingWorkUnitBuilder(WorkUnitBuilder):
    """
    Used to construct a WorkUnit with its necessary components.
    Constructs CandidatesWorkUnits for the process candidates task.
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(VettingWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):
        return VettingWorkUnit(
            filename, data, progress_manager, output_context,
            dry_run=dry_run)

class ExamineWorkUnitBuilder(WorkUnitBuilder):
    """
    Used to construct a WorkUnit with its necessary components.
    Constructs CandidatesWorkUnits for the process candidates task.
    """

    def __init__(self, parser, input_context, output_context, progress_manager,
                 dry_run=False):
        super(ExamineWorkUnitBuilder, self).__init__(
            parser, input_context, output_context, progress_manager,
            dry_run=dry_run)

    def _do_build_workunit(self,
                           filename,
                           data,
                           progress_manager,
                           output_context,
                           dry_run):
        return ExamineWorkUnit(
            filename, data, progress_manager, output_context,
            dry_run=dry_run)

class ExamineWorkUnit(RealsWorkUnit):
    pass


import os
from dotenv import load_dotenv
import requests
from pprint import pprint

load_dotenv()

API_KEY = os.getenv("API_KEY")
API_URL = os.getenv("API_URL")

url = f"{API_URL}/indexes"

INDEX_NAME = "test_index"

headers = {"x-api-key": API_KEY}

data = {
    "engine_id": "marengo2.5",
    "index_options": ["visual", "conversation", "text_in_video", "logo"],
    "index_name": INDEX_NAME,
}

response = requests.post(url, headers=headers, json=data)
INDEX_ID = response.json().get("_id")

print(f"Status code: {response.status_code}")
pprint(response.json())


#!/usr/bin/python3
"""script that lists all State objects that contain the
letter a from the database
"""

import sys
from model_state import Base, State
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

if __name__ == "__main__":

    engine = create_engine('mysql+mysqldb://{}:{}@localhost/{}'.format(
                           sys.argv[1], sys.argv[2], sys.argv[3]),
                           pool_pre_ping=True)
    Session = sessionmaker(bind=engine)

    session = Session()
    state = session.query(State).filter(State.name.contains('a'))

    if state is not None:
        for stat in state:
            print("{}: {:s}".format(stat.id, stat.name))


from django.urls import path
from . import views

urlpatterns = [
    path('', views.loadMain ,name = "indexpage"),
    path('login/',views.login , name= "login"),
    path('signin/',views.singin , name = "sign"),
    path('dashboard/',views.userDashBroard,name='userdashbord'),
    path('logout/<int:id>',views.logout,name='logout'),
    path('selectsubject/',views.userDashBrod_selectedsub,name='selecet_subject'),
    path('addusersubject/',views.addusersubject,name='adduser_subject'),
    path('removeusersubject/<int:id>',views.removeusersubject,name='removeuser_subject'),
    path('exampage/', views.exampage, name='exampage'),
    path('dashboard/json/', views.json, name='test_json'),
    path('dashboard/json_subject',views.json_subject,name='json_subject'),
    path('exampage/sub/<int:id>',views.examsubtopic,name="examsubtopic"),#json
    path('exampage/sub/subexam/<int:id>',views.examsubtopicpage , name ="examsubtopicpage"), # real page
    path('exampage/check',views.testingPost,name="check"),
    path('exampage/exam_json',views.getExamResult,name='exam_json'),
    # path('exampage/exam_chart/<int:id>', views.getSubjectChart, name='exam_chart'),   # delete this code
    path('exampage/json/<int:id>',views.getSubjectChart , name='exam_sub_json'), # get json informatoin
    path('exampage/json_chart/<int:id>', views.getChart, name='exam_chart'),
    path('exampage/fullpage/<int:id>',views.fullpaper,name='fullpapaer'),
    path('dashboard/profile',views.updateUser,name='userprofile'),
]


import cv2 as cv
import numpy as np

img = cv.imread('Resources/Photos/park.jpg')
cv.imshow('Park', img)

gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
cv.imshow('Gray', gray)

#simple thresholding
threshold,thresh = cv.threshold(gray,100,256,cv.THRESH_BINARY)
cv.imshow('simpleThresh',thresh)

threshold,thresh_inv = cv.threshold(gray,100,256,cv.THRESH_BINARY_INV)
cv.imshow('simpleThresh_inv',thresh_inv)

#adaptive thresh
adaptive = cv.adaptiveThreshold(gray,255,cv.ADAPTIVE_THRESH_MEAN_C,cv.THRESH_BINARY,11,3)
cv.imshow('adaptive',adaptive)

cv.waitKey(0)


import os
import pandas as pd
from sklearn.model_selection import train_test_split

cdir = os.path.dirname(os.path.abspath(__file__))
news = pd.read_table(cdir + '/newsCorpora.csv', header=None)
news.columns = ['ID', 'TITLE', 'URL', 'PUBLISHER', 'CATEGORY', 'STORY', 'HOSTNAME', 'TIMESTAMP']

# Reuters”, “Huffington Post”, “Businessweek”, “Contactmusic.com”, “Daily Mail”の5社のみを対象とする
specific_news = news[news['PUBLISHER'].isin(
    ['Reuters', 'Huffington Post', 'Businessweek', 'Contactmusic.com', 'Daily Mail'])]

# TITLEとCATEGORYの列のみを抽出する
specific_news = specific_news[['TITLE', 'CATEGORY']]

# データをシャッフルする
# q: このrandom_stateは何？
# a: 乱数のシード値。同じ値を指定すると、同じ乱数が生成される。
# q: このfracは何？
# a: 0.5なら、半分のデータを返す。1なら、全てのデータを返す。
specific_news = specific_news.sample(frac=1, random_state=0)

# データを学習用と評価用に分割する
# stratify: ラベルの比率を維持する
train_data, valid_data = train_test_split(specific_news, test_size=0.2, random_state=0, stratify=specific_news['CATEGORY'])

# 評価用データを評価用とテスト用に分割する
valid_data, test_data = train_test_split(valid_data, test_size=0.5, random_state=0, stratify=valid_data['CATEGORY'])

# データの保存
train_data.to_csv(cdir + '/train.txt', sep='\t', index=False)
valid_data.to_csv(cdir + '/valid.txt', sep='\t', index=False)
test_data.to_csv(cdir + '/test.txt', sep='\t', index=False)


import logging
from argparse import ArgumentParser
from pathlib import Path
from typing import List
import numpy as np
from tqdm import tqdm
import pandas as pd

def stack_features(features_files: List[Path]):
    data = [np.load(str(fl)) for fl in features_files]
    sample_length = max([ft.shape[0] for ft in data])
    for i in range(len(data)):
        if data[i].shape[0] < sample_length:
            paddings = np.zeros((sample_length - data[i].shape[0], data[i].shape[1]))
            data[i] = np.concatenate([data[i], paddings], axis=0)

    return np.concatenate(data, axis=-1)

if __name__ == '__main__':

    arg_parser = ArgumentParser()
    arg_parser.add_argument("--motion", action="store", type=str, nargs='*', help="List of folders with motion data")
    arg_parser.add_argument("--audio", type=str, nargs='*', help="List of folders with audio data")
    arg_parser.add_argument("--phase", type=str, nargs='*', help="List of folders with phase data")
    arg_parser.add_argument("--dst", type=str, help="Folder to store resulted dataset")
    arg_parser.add_argument("--first", action="store_true")
    arg_parser.add_argument("--metadata_trn_path", type=str, default=None)
    args = arg_parser.parse_args()

    dst_folder = Path(args.dst)
    if not dst_folder.exists():
        dst_folder.mkdir(parents=True)
    logging.basicConfig(level=logging.INFO, filename=str(dst_folder / "log.txt"))

    motion_folders = [Path(folder) for folder in args.motion] if args.motion is not None else []
    audio_folders = [Path(folder) for folder in args.audio]
    phase_folders = [Path(folder) for folder in args.phase] if args.phase is not None else []
    logging.info(f"Motion: {motion_folders}\n"
                 f"Audio: {audio_folders}\n"
                 f"Phase: {phase_folders}")

    first_file_names = []
    if args.first:
        assert args.metadata_trn_path is not None, "You must specify the path to the thain metadata"
        metadata_trn = pd.read_csv(args.metadata_trn_path)
        for i in range(len(metadata_trn)):
            file_name = metadata_trn["prefix"].iloc[i]
            if metadata_trn["main-agent_id"].iloc[i] == 1:
                first_file_names.append(file_name)

    for audio_path in tqdm(audio_folders[0].glob('*.npy')):
        if args.first:
            cur_name = audio_path.name.replace("*.npy", "")
            if cur_name not in first_file_names:
                continue

        audio_files = [audio_folder / audio_path.name for audio_folder in audio_folders]
        audio_data = stack_features(audio_files)

        motion_data = None
        if len(motion_folders) > 0:
            motion_files = [motion_folder / audio_path.name for motion_folder in motion_folders]
            motion_data = stack_features(motion_files)
        phase_data = None
        if len(phase_folders) > 0:
            phase_files = [phase_folder / audio_path.name for phase_folder in phase_folders]
            phase_data = stack_features(phase_files)

        dst_path = dst_folder / audio_path.name.replace('.npy', '.npz')
        logging.info(f"{dst_path}:\n"
                     f"Motion shape: {motion_data.shape if motion_data is not None else None}\n"
                     f"Audio shape: {audio_data.shape}\n"
                     f"Phase shape: {phase_data.shape if phase_data is not None else None}\n")

        np.savez(str(dst_path), Audio=audio_data, Motion=motion_data, Phase=phase_data)


# -*- coding: utf-8 -*-

import logging
import optparse
import StringIO

from django.core.management.base import BaseCommand
from django.db.utils import IntegrityError
from django.template.defaultfilters import slugify

from easy_thumbnails.exceptions import InvalidImageFormatError
from easy_thumbnails.files import get_thumbnailer

from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import CourseLocator
from openedx.core.djangoapps.models.course_details import CourseDetails
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.django import modulestore

from courses import settings as courses_settings
from courses.models import Course, CourseUniversityRelation
from universities.models import University

logger = logging.getLogger(__name__)

class CourseHandler(object):

    def __init__(self, course):
        self.key = unicode(course.id)
        self.course = course

    @property
    def memory_image_file(self):
        try:
            asset_location = StaticContent.get_location_from_path(self.image_url)
            content = contentstore().find(asset_location, as_stream=True)
        except (NotFoundError, InvalidKeyError):
            return None
        image_file = StringIO.StringIO(content.copy_to_in_mem().data)
        return image_file

    @property
    def title(self):
        title = self.course.display_name_with_default
        return title or ''

    @property
    def university_name(self):
        name = ''
        if self.course.display_organization:
            name = unicode(self.course.display_organization)
        return name

    @property
    def image_location(self):
        course_locator = CourseLocator.from_string(self.key)
        location = StaticContent.compute_location(
            course_locator, self.course.course_image
        )
        return location

    @property
    def image_url(self):
        location = self.image_location
        url = unicode(location)
        return url

    def make_thumbnail(self, options):
        if not self.memory_image_file:
            return None
        base_filename = slugify(self.key)
        try:
            thumbnail = get_thumbnailer(
                self.memory_image_file,
                relative_name='courses-thumbnails/{}'.format(base_filename)
            ).get_thumbnail(options)
        except (InvalidImageFormatError, OSError) as err:
            logger.error(err)
            thumbnail = None
        return thumbnail

    def get_university(self):
        try:
            university = University.objects.get(code=self.course.org)
        except University.DoesNotExist:
            university = None
        return university

    def assign_university(self, course):
        university = self.get_university()
        assigned = False
        if university:
            try:
                CourseUniversityRelation.objects.create(
                    course=course,
                    university=university
                )
                assigned = True
            except IntegrityError:
                pass
        return assigned

class Command(BaseCommand):
    help = "Update FUN's course data."
    option_list = BaseCommand.option_list + (
        optparse.make_option(
            '--force-universities-assignment',
            action='store_true',
            dest='assign_universities',
            default=False,
            help='Force university assignment on existing courses.',
        ),
        optparse.make_option(
            '--course-id',
            action='store',
            type='string',
            dest='course_id',
            default='',
            help='Update only the given course.',
        ),
    )

    def update_all_courses(self, mongo_courses, assign_universities=False):
        '''
        For each course, we create or update the corresponding
        course in SQL Course table.
        '''
        for mongo_course in mongo_courses:
            try:
                self.update_course(
                    modulestore().get_course(mongo_course.id),
                    assign_universities=assign_universities
                )
            except InvalidKeyError as err:
                # Log the error but continue indexing other courses
                logger.error(err)

        self.stdout.write('Number of courses parsed: {}\n'.format(len(mongo_courses)))
        return None

    def update_course(self, mongo_course, assign_universities=False):
        '''
        For the given course, we create or update the corresponding
        course in SQL Course table.
        '''

        course_handler = CourseHandler(mongo_course)
        key = course_handler.key
        self.stdout.write('Updating data for course {}\n'.format(key))
        course, was_created = Course.objects.get_or_create(key=key)
        if was_created or assign_universities:
            university = course_handler.assign_university(course)
            if university:
                self.stdout.write('\t University assigned to "{}"\n'.format(key))
            else:
                self.stdout.write('\t No university assigned to "{}"\n'.format(key))
        course.is_active = True
        course.university_display_name = course_handler.university_name
        course.title = course_handler.title
        course.image_url = course_handler.image_url
        thumbnails_info = {}
        for thumbnail_alias, thumbnails_options in \
                courses_settings.FUN_THUMBNAIL_OPTIONS.items():
            thumbnail = course_handler.make_thumbnail(thumbnails_options)
            if thumbnail:
                thumbnails_info[thumbnail_alias] = thumbnail.url
        course.thumbnails_info = thumbnails_info
        course.start_date = mongo_course.start
        course.enrollment_start_date = mongo_course.enrollment_start
        course.enrollment_end_date = mongo_course.enrollment_end
        course.end_date = mongo_course.end

        course_key = CourseKey.from_string(key)
        course.short_description = CourseDetails.fetch_about_attribute(
            course_key,
            'short_description',
        )

        course.save()
        del course
        self.stdout.write('Updated course {}\n'.format(key))
        return None

    def deactivate_orphan_courses(self, mongo_courses):
        '''
        Orphan courses are entries that exist in SQL Course table while the
        corresponding Mongo courses are not listed anymore. This method deactivate
        orphan courses.
        '''
        courses_keys = [unicode(c.id) for c in mongo_courses]
        orphan_courses = Course.objects.exclude(key__in=courses_keys)
        orphan_courses.update(is_active=False)
        self.stdout.write('Deactivated {} orphan courses\n'.format(orphan_courses.count()))
        return None

    def handle(self, *args, **options):
        '''
        This command can handle the update of a single course if the course ID
        if provided. Otherwise, it will update all courses found in MongoDB.
        '''
        assign_universities = options.get('assign_universities')
        course_id = options.get('course_id')
        if course_id:
            # course_key is a CourseKey object and course_id its sting representation
            course_key = CourseKey.from_string(course_id)
            course = modulestore().get_course(course_key)
            self.update_course(
                mongo_course=course,
                assign_universities=assign_universities
            )
        else:
            courses = modulestore().get_courses()
            self.update_all_courses(
                mongo_courses=courses,
                assign_universities=assign_universities,
            )
            self.deactivate_orphan_courses(courses)


#!/usr/bin/python

from musicazoo.lib.mzbot import MZBot
import time
import sqlite3
import os
import re

class TrackerBot(MZBot):
	def __init__(self,url):
		self.luid=None
		self.db=os.path.join(os.path.dirname(os.path.realpath(__file__)),'mz.db')
		MZBot.__init__(self,url)

	def poll(self):
		req={'cmd':'cur','args':{'parameters':{'youtube':['url']}}}
		result=self.assert_success(self.doCommand(req))
		if result and result['type']=='youtube':
			uid=result['uid']
			if uid != self.luid:
				url=result['parameters']['url']
				v=re.search(r'watch\?v=(.+)$',url)
				if v:
					self.got(v.group(1))
				self.luid=uid
		else:
			self.luid=None

	def got(self,url):
		conn=sqlite3.connect(self.db)
		c=conn.cursor()
		c.execute('INSERT INTO youtube_history (url) VALUES (?)',[url])
		conn.commit()
		conn.close()

if __name__ == "__main__":
	tb = TrackerBot("http://musicazoo.mit.edu/cmd")
	while True:
		time.sleep(3)
		tb.poll()


import hdl
reload(hdl)

from hdl.models import SparseSlowModel
from hdl.learners import SGD

whitenpatches = 150000

#l = SGD(model=SparseSlowModel(patch_sz=48,N=2048,T=48,sparse_cost='subspacel1',slow_cost='dist'),datasource='PLoS09_Cars_Planes',batchsize=48,display_every=20000,save_every=20000,eta_target_maxupdate=.05)
#l = SGD(model=SparseSlowModel(patch_sz=32,N=1028,T=64,sparse_cost='subspacel1',slow_cost='dist'),datasource='PLoS09_Cars_Planes',batchsize=64,display_every=20000,save_every=20000,eta_target_maxupdate=.05)
l = SGD(model=SparseSlowModel(patch_sz=20,N=400,T=48,sparse_cost='subspacel1',slow_cost='dist',perc_var=99.9),datasource='PLoS09_Cars_Planes',batchsize=48,display_every=1000,save_every=20000,eta_target_maxupdate=.05)

databatch = l.get_databatch(whitenpatches)
l.model.learn_whitening(databatch)
l.model.setup()

from kairos.data import PLoS09
#from kairos.metrics.kanalysis import kanalysis

from devthor.kanalysis import kanalysis_linear as kanalysis

X, Y = PLoS09.get_testing_selection(l.videos,l.chunk_labels)

def evaluate_kernel(model,X,Y,output_function='infer_abs'):

    offset_y = (X.shape[1] - model.patch_sz)/2
    offset_x = (X.shape[2] - model.patch_sz)/2
    X = X[:,offset_y:offset_y+model.patch_sz,offset_x:offset_x+model.patch_sz]

    batch = X.reshape(X.shape[0],X.shape[1]*X.shape[2]).T

    new_X = model.output(model.preprocess(batch),output_function=output_function).T

    k_curve, k_auc = kanalysis(new_X,Y)

    return k_curve, k_auc

def plot_kanalysis(l,X,Y):

    from matplotlib import pyplot as plt
    import os
    from hdl.config import state_dir

    output_functions = ['infer', 'infer_abs', 'proj', 'proj_abs', 'infer_loga', 'proj_loga']

    all_k_auc = []
    for output_function in output_functions:
        k_curve, k_auc = evaluate_kernel(l.model,X,Y,output_function)

        all_k_auc.append(k_auc)

        plt.figure(10)
        plt.clf()
        plt.plot(range(len(k_curve)),k_curve)
        plt.xlabel('Kernel dim')
        plt.ylabel('loss')
        plt.title('Kernel Analysis: %.4f'%k_auc)

        savepath = os.path.join(state_dir,l.model.model_name + '_' + l.model.tstring)
        if not os.path.exists(savepath): os.makedirs(savepath)

        fname = os.path.join(savepath,'KAnalysis_%s_%d.png'%(output_function,l.iter))
        plt.savefig(fname)

    return l.iter, min(all_k_auc)

def plot_k_results(k_results):

    from matplotlib import pyplot as plt
    import os
    from hdl.config import state_dir
    import numpy as np

    trials = np.array([k[0] for k in k_results])
    values = np.array([k[1] for k in k_results])

    plt.figure(10)
    plt.clf()
    plt.plot(trials,values)
    plt.xlabel('Save trial')
    plt.ylabel('KAnalysis AUC')
    plt.title('Kernel Analysis AUC')

    savepath = os.path.join(state_dir,l.model.model_name + '_' + l.model.tstring)
    if not os.path.exists(savepath): os.makedirs(savepath)

    fname = os.path.join(savepath,'KAnalysis_AUC_progress.png')
    plt.savefig(fname)

k_results = []

k_results.append(plot_kanalysis(l,X,Y))
plot_k_results(k_results)

epochs = 20
for epoch in range(epochs):
    l.learn(iterations=10000)
    k_results.append(plot_kanalysis(l,X,Y))
    plot_k_results(k_results)

l.change_target(.1)
epochs = 10
for epoch in range(epochs):
    l.learn(iterations=10000)
    k_results.append(plot_kanalysis(l,X,Y))
    plot_k_results(k_results)

l.change_target(.1)
epochs = 10
for epoch in range(epochs):
    l.learn(iterations=10000)
    k_results.append(plot_kanalysis(l,X,Y))
    plot_k_results(k_results)

from hdl.display import display_final
display_final(l.model)