#!/usr/bin/env python
# -*- coding: utf-8 -*-
__date__= 'Dec 04, 2013 '
__author__= 'samuel'

import logging
log = logging.getLogger(__name__)

import inspect
import traceback

from petsquarebackend.services import BaseService
from petsquarebackend.services import ServiceMethod
from petsquarebackend.models.location import Location_TB

class LocationService(BaseService):
    def __init__(self, request):
        super(LocationService, self).__init__('LocationService', request)

    @ServiceMethod
    def list(self, user_id=None, offset=0, size=100):
        status = self.status.copy()
        if user_id:
            success, models = Location_TB.list(filattr=('explorer_id', user_id),
                                      offset=offset,
                                      size=size)
        else:
            success, models = Location_TB.list(offset=offset,
                                      size=size)

        status = self.serv_rtn(status=status, success=success, model=models)
        status['info']['count'] = len(models)
        return status

    @ServiceMethod
    def create(self, name, description, longitude, latitude, address, user_id):
        status = self.status.copy()
        success, model = Location_TB.create(name=name,
                                            description=description,
                                            longitude=longitude,
                                            latitude=latitude,
                                            address=address,
                                            explorer_id=user_id)
        status = self.serv_rtn(status=status, success=success, model=model)
        return status

    @ServiceMethod
    def show(self, id):
        status = self.status.copy()
        success, model = Location_TB.show(id)
        status = self.serv_rtn(status=status, success=success, model=model)
        return status

    @ServiceMethod
    def update(self, id, data):
        status = self.status.copy()
        success, model = Location_TB.update(id=id,
                           name=data['name'],
                           description=data['description'],
                           longitude=data['longitude'],
                           latitude=data['latitude'],
                           address=data['address'],
                           explorer_id=data['user_id'],)
        status = self.serv_rtn(status=status, success=success, model=model)
        return status

    @ServiceMethod
    def delete(self, id):
        status = self.status.copy()
        success, model = Location_TB.delete(id=id)
        status = self.serv_rtn(status=status, success=success, model=model)
        return status

    def search_latlng(self, user_id, latitude, longitude, radius=0.00449661, size=100):
        status = self.status.copy()
        success, models = Location_TB.search_latlng_with_radius(latitude,
                                                       longitude,
                                                       radius,
                                                       size)
        status = self.serv_rtn(status=status, success=success, model=models)
        models_len = len(models)
        status['info']['count'] = models_len
        if models_len > size:
            status['info']['msg'] = 'radius too large.'
        return status

def main():
    pass

if __name__ == '__main__':
    main()


import time
import conf

import community as nxlouvain
# import matplotlib.pyplot as plt
import networkx as nx
import pandas as pd
from dateutil import rrule
import json

from classes.neocontroller import Neo4jController

def analyze_repos(owners):

    for owner in owners:
        analyzer = Analyzer(owner)
        analyzer.export_graphjson()
    else:
        pass

class Analyzer:
    def __init__(self,
                 owner: str,
                 repo=None):
        self._owner = owner
        self._repo = repo

        self._controller = Neo4jController()

        self._startdt, self._enddt = self._controller.get_comment_timeframe(self._owner)

        self._degree_centrality = None
        self._betweenness_centrality = None
        self._eigenvector_centrality = None
        self._partition = None
        self._modularity = None

        self.run()

    def run(self):
        self._individual_measures()
        self._export_measures()

    def get_groups(self):
        return self._partition

    def get_betweenness_centrality(self):
        return self._betweenness_centrality

    def get_degree_centrality(self):
        return self._degree_centrality

    def get_modularity(self):
        return self._modularity

    def _louvain_networkx(self):

        print("Running NX Louvain algorithm for {0} and timeframe length {1}".format(self._owner,
                                                                                conf.a_length_timeframe))
        res = {}
        for dt in rrule.rrule(rrule.WEEKLY, dtstart=self._startdt, until=self._enddt):
            time_start = time.time()

            links = self._controller.get_communication_subgraph(self._owner, dt)

            if not links.empty:
                nxgraph = nx.from_pandas_dataframe(links, source="source", target="target", create_using=nx.MultiGraph())

                partition = nxlouvain.best_partition(nxgraph)

                # partition = self.convert_keys_to_string(partition)

                res[dt.strftime("%Y-%m-%d")] = partition

            print("current: {0} - time: {1}".format(dt.date(), time.time() - time_start))

        return res

    def _individual_measures(self):

        print("Running NX analysis for {0} and timeframe length {1}".format(self._owner,
                                                                            conf.a_length_timeframe))
        res_louvain = {}
        res_degree_centrality = {}
        res_betweenness_centrality = {}
        res_eigenvector_centrality = {}
        res_modularity = {}

        time_start = time.time()
        for dt in rrule.rrule(rrule.WEEKLY, dtstart=self._startdt, until=self._enddt):
            lap_time = time.time()

            links = self._controller.get_communication_subgraph(self._owner, dt)

            if not links.empty:
                multi_graph = nx.from_pandas_dataframe(links,
                                                       source="source",
                                                       target="target",
                                                       create_using=nx.MultiGraph())

                if conf.a_louvain:
                    partition = nxlouvain.best_partition(multi_graph)
                    res_louvain[dt.strftime("%Y-%m-%d")] = partition

                if conf.a_betweenness_centrality:
                    bc = nx.betweenness_centrality(multi_graph, normalized=True)
                    res_betweenness_centrality[dt.strftime("%Y-%m-%d")] = bc

                if conf.a_degree_centrality:
                    dc = nx.degree_centrality(multi_graph)
                    res_degree_centrality[dt.strftime("%Y-%m-%d")] = dc

                if conf.a_modularity:
                    mod = nxlouvain.modularity(partition, multi_graph)
                    res_modularity[dt.strftime("%Y-%m-%d")] = mod

                if conf.a_eigenvector_centrality:
                    pass
                    # simple_graph = self.convert_to_simple(multi_graph)

                    # TODO: eigenvector centrality calculation fails occasionally
                    # reason may be that nx.eigenvector_centrality() can't handle star graphs
                    # https://stackoverflow.com/questions/43208737/using-networkx-to-calculate-eigenvector-centrality
                    # ec = nx.eigenvector_centrality(simple_graph)
                    # res_eigenvector_centrality[dt.strftime("%Y-%m-%d")] = ec

            if conf.output_verbose:
                print("current: {0} - time: {0:.2f}s".format(dt.date(), time.time() - lap_time))

        print("{0:.2f}s".format(time.time()-time_start))
        print()

        self._modularity = res_modularity
        self._degree_centrality = res_degree_centrality
        self._betweenness_centrality = res_betweenness_centrality
        self._degree_centrality = res_degree_centrality
        self._eigenvector_centrality = res_eigenvector_centrality
        self._partition = res_louvain

    def _export_measures(self):
        if conf.a_betweenness_centrality:
            pd.DataFrame.from_dict(self._betweenness_centrality)\
                .to_csv(conf.get_nx_path(self._owner, "bc",  self._repo))

        if conf.a_degree_centrality:
            pd.DataFrame.from_dict(self._degree_centrality)\
                .to_csv(conf.get_nx_path(self._owner, "dc", self._repo))

        if conf.a_eigenvector_centrality:
            pd.DataFrame.from_dict(self._eigenvector_centrality) \
                .to_csv(conf.get_nx_path(self._owner, "ec", self._repo))

        if conf.a_modularity:
            i = "mod"
            df = pd.DataFrame.from_dict(self._modularity, orient="index")
            df.to_csv(conf.get_nx_path(self._owner, i, self._repo))
            if conf.a_gen_charts:
                pass
                # df.plot(title="Modularity ({0}/{1})".format(self._owner, self._repo),
                #         legend=False)
                #  plt.savefig(conf.get_plot_path(self._owner, self._repo, i))

    def convert_keys_to_string(self, o):
        """
        Recursively converts dictionary keys to strings. Type conversion is required for exporting JSON files.

        :param o:      any object. If o is dictionary, method calls itself
        :return:       --
        """

        if not isinstance(o, dict):
            return o
        return dict((str(k), self.convert_keys_to_string(v))
                    for k, v in o.items())

    @staticmethod
    def convert_to_simple(multi_graph):
        simple_graph = nx.Graph()
        for u, v, data in multi_graph.edges(data=True):
            w = data['weight'] if 'weight' in data else 1.0

            if simple_graph.has_edge(u, v):
                simple_graph[u][v]['weight'] += w
            else:
                simple_graph.add_edge(u, v, weight=w)

        return simple_graph

    def export_graphjson(self):

        """
        Exports link data for single repositories as JSON files suitable to serve as input for  the gitNetViewer tool.

        Projects and export links to be exported can be configured in the conf module.

        The resulting JSON string has the following fields:

            "info":             a subset of repository related information fields containing
                                    owner:          owner name
                                    repo:           repository name
                                    total_links:    total number of links
                                    total_nodes:    total number of nodes
                                    no_threads:     total number of threads
                                    no_comments:    total number of comments

            "nodes":            a list of nodes with the fields
                                    name:       lowercase string representation of the users' login names
                                    id:         node id as provided by the database

            "links":            a list of links with the fields
                                    source:         the source node's id as provided by the database
                                    target:         the target node's id as provided by the database
                                    rel_type:       the relation type
                                    timestamp:      timestamp in the format 'yyyy-MM-dd'
                                    link_id:        link id as provided by the database

            "groups":           group attribution per node over time as provided by the Analyzer class
            "d_centrality":     degree centrality per node over time as provided by the Analyzer class
            "b_centrality":     betweenness centrality per node over time as provided by the Analyzer class
            "modularity":       modularity over time as provided by the Analyzer class

        :return:    --
        """

        print("exporting data in graphJSON format for {0}".format(self._owner))

        groups = self.convert_keys_to_string(self._partition)
        d_centrality = self.convert_keys_to_string(self._degree_centrality)
        b_centrality = self.convert_keys_to_string(self._betweenness_centrality)
        modularity = self.convert_keys_to_string(self._modularity)

        nodes, links, info = self._controller.get_viz_data(self._owner)

        data = {"info": info,
                "nodes": nodes,
                "links": links,
                "groups": groups,
                "d_centrality": d_centrality,
                "b_centrality": b_centrality,
                "modularity": modularity}

        with open(conf.get_viz_data_path(self._owner, self._repo), "w") as fp:
            json.dump(data, fp, indent="\t")

if __name__ == '__main__':
    owners = [
        'OneDrive',
        'waffleio',
        'getnikola',
        'Tribler',
        'BobPalmer',
        'novus',
        'rathena',
        'gatsbyjs']

    # repos = [16727251]
    analyze_repos(owners)


import dash
import dash_cytoscape as cyto
import dash_html_components as html

# Dashインスタンスの生成
app = dash.Dash(__name__)

# Cytoscapeコンポーネントの生成
cyto_compo = cyto.Cytoscape(
    id="hello-dash-cyto",
    style={"width": "400px", "height": "400px"},
    # ネットワークの構成要素の定義
    elements=[
        # ノードの定義
        {"data": {"id": "A", "name": "Alice"}},
        {"data": {"id": "B", "name": "Bob"}},
        # エッジの定義
        {"data": {"source": "A", "target": "B",
                  "name": "family",
                  "source_name": "parent", "target_name": "child"}},
    ],
    # ノード配置方法の定義
    layout={"name": "grid"},
    # スタイルの定義
    stylesheet=[
        # ❶ ノード全体のラベルを変更
        {"selector": "node",
         "style": {"label": "data(name)",  # ラベル文字列
                   "color": "white",  # 文字色を白
                   "text-background-color": "green",  # 背景色を緑
                   "text-background-opacity": 1,  # 完全に不透明
                   "text-halign": "right",  # ノードの右側
                   "text-valign": "center",}}, # ノードの中央
        # ❷ エッジ全体のラベルを変更
        {"selector": "edge",
         "style": {"content": "data(name)",  # ラベル文字列
                   "source-label": "data(source_name)",  # 始点側のラベル文字列
                   "target-label": "data(target_name)",  # 終点側のラベル文字列
                   "source-text-offset": 25,  # 始点側のラベルの位置
                   "target-text-offset": 25}},  # 終点側のラベルの位置
    ],
)

# Cytoscapeコンポーネントをレイアウトに渡す
app.layout = html.Div([cyto_compo])

if __name__ == "__main__":
    # サーバの起動
    app.run_server(debug=True)


#!/usr/bin/env python
# -*- coding: ascii -*-
from __future__ import print_function
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import str
from builtins import range
from builtins import object

import random

DEALER_HITS_BELOW = 17

class Hand( object ):
    """Either the dealer or player hand"""
    def __init__(self, name='???', auto_fill_to_12=False):
        self.name = name
        self.new_hand(auto_fill_to_12=auto_fill_to_12)

    def new_hand(self, auto_fill_to_12=False): # can call directly for a new hand
        self.cardL = []
        self.usable_ace = False
        self.get_card(2)

        if auto_fill_to_12:
            while self.summ < 11:
                self.get_card(1)

    def get_card(self, N=1):
        for i in range(N):
            self.cardL.append( random.choice( (1,2,3,4,5,6,7,8,9,10,10,10,10) ) )
        self.calc_summ()

    def get_show_card(self):
        return self.cardL[0] # use 1st card as show card.

    def set_player_hand(self, summ=0, usable_ace=False ):
        if usable_ace:
            self.cardL = [1, summ-11]
        else:
            c1 = random.choice( tuple( range(2, summ-1, 1) ) )
            c2 = summ - c1
            self.cardL = [c1, c2]

        self.calc_summ()

    def set_dealer_hand(self, dealer_showing=0 ):
        self.cardL = [dealer_showing]
        self.get_card(1)

    def calc_summ(self):
        self.usable_ace = False

        self.summ = sum( self.cardL )
        if 1 in self.cardL:
            if (self.summ <= 11):
                self.summ += 10 # use the ace as an 11
                self.usable_ace = True

        self.is_bust = (self.summ > 21)

        self.is_natural = (len(self.cardL)==2) and (self.summ==21)

    def summ_print(self):
        s = ''
        if self.is_bust:
            s = '  BUSTED'*4
        elif self.is_natural:
            s = '  NATURAL 21'
        elif self.usable_ace:
            s = '  USABLE ACE'

        print('___%s Hand Summary___'%self.name)
        print('    Cards = ', ', '.join( ['%s'%i for i in self.cardL] ))
        print('    Summ  =', self.summ, s)

class BlackJack( object ):

    def __init__(self):
        self.deal_hands()

    def deal_hands(self):
        self.playerH = Hand('Player', auto_fill_to_12=True)
        self.dealerH = Hand('Dealer')
        self.game_over = False
        self.dealer_is_done = False
        self.reward = 0.0
        self.evaluate()

    def re_deal(self):
        self.playerH.new_hand( auto_fill_to_12=True )
        self.dealerH.new_hand()
        self.game_over = False
        self.dealer_is_done = False
        self.reward = 0.0
        self.evaluate()

    def evaluate(self):
        """
        Only evaluates the player move.
        Final reward happens when dealer plays.
        """
        if self.playerH.is_bust:
            self.reward = -1.0
            self.game_over = True

        elif self.dealerH.is_bust:
            self.reward = 1.0
            self.game_over = True

        elif self.playerH.is_natural:
            self.game_over = True

            if self.dealerH.is_natural:
                self.reward = 0.0
            else:
                self.reward = 1.0 # player wins outright.

    def player_hits(self):
        self.playerH.get_card()
        self.evaluate()

    def dealer_plays(self):

        while (not self.game_over) and (self.dealerH.summ < DEALER_HITS_BELOW):
            #print('--> Dealer Hits at:', self.dealerH.summ, end=' ')
            self.dealerH.get_card()
            self.evaluate()
            #print('Card is:', self.dealerH.cardL[-1], 'game_over =',self.game_over)

        self.dealer_is_done = True

        # need to decide winner based on sum of cards
        if not self.game_over: # i.e. nobody busted, so need to compare sums.
            self.game_over = True

            if self.dealerH.summ > self.playerH.summ:
                self.reward = -1.0
            elif self.dealerH.summ < self.playerH.summ:
                self.reward = 1.0
            else:
                self.reward = 0.0

    def get_state_hash(self):
        """State Hash is (player sum, usable ace flage, dealer show card)"""
        return (self.playerH.summ, self.playerH.usable_ace, self.dealerH.get_show_card() )

    def set_state_hash(self, state_hash ):
        (player_sum, usable_ace, dealer_showing) = state_hash

        self.playerH.set_player_hand( summ=player_sum, usable_ace=usable_ace )
        self.dealerH.set_dealer_hand( dealer_showing=dealer_showing )

        self.game_over = False
        self.dealer_is_done = False
        self.reward = 0.0
        self.evaluate()

    def summ_print(self):
        print('___BlackJack___  state_hash =', self.get_state_hash() )
        if self.game_over:
            print('***GAME OVER*** Reward=',self.reward, end=' ')

        if self.reward > 0.1:
            print('PLAYER WINS')
        elif self.reward < -0.1:
            print('PLAYER LOSES')
        else:
            print('DRAW GAME')

        self.playerH.summ_print()
        self.dealerH.summ_print()

def play_policy_game( BJ, policyD ):

    if not BJ.game_over:
        action = policyD[ BJ.get_state_hash() ]

        while action == 'Hit':
            #print('--> Player Hits at:', BJ.playerH.summ, end=' ')
            BJ.player_hits()
            #print('Card is:', BJ.playerH.cardL[-1],
            #      'Hand is:',BJ.playerH.cardL,
            #      '  game_over=',BJ.game_over)

            if BJ.game_over:
                action = 'Stay'
            else:
                if BJ.get_state_hash() not in policyD:
                    print('ERROR... BJ.get_state_hash() =',BJ.get_state_hash())
                action = policyD[ BJ.get_state_hash() ]

    if not BJ.game_over:
        BJ.dealer_plays()

def mock_game( BJ ):
    if not BJ.game_over:
        if BJ.playerH.summ < 17:
            print('--> Player Hits at:', BJ.playerH.summ, end=' ')
            BJ.player_hits()
            print('Card is:', BJ.playerH.cardL[-1], '  game_over=',BJ.game_over)

    if not BJ.game_over:
        BJ.dealer_plays()

if __name__ == "__main__":

    BJ = BlackJack()
    mock_game( BJ )
    BJ.summ_print()
    print('-'*55)

    BJ.re_deal()
    mock_game( BJ )
    BJ.summ_print()
    print('-'*55)

    BJ.set_state_hash( (11, False, 5) )
    mock_game( BJ )
    BJ.summ_print()


from data.data import load_data_config_dict

def process_regression_models(load_data_config_dict):
    models = load_data_config_dict['models']

    models_dict = {}
    for model_name, model in models.items():
        model_dict = {}
        variables = []
        correction_variable = None
        description = ''
        training = {}

        if 'variables' in model:
            variables.extend(model['variables'].keys())
        if 'differential variables' in model:
            variables.extend(model['differential variables'].keys())
        if 'correction variable' in model:
            correction_variable = list(model['correction variable'].keys())[0]
        if 'description' in model:
            description = model['description']
        if 'training' in model:
            training = model['training']

        model_dict = dict(variables=variables, correction_variable=correction_variable,
        description=description, training=training)
        models_dict[model_name] = model_dict
    return models_dict

regression_models = process_regression_models(load_data_config_dict)


import datetime
import os
import sys
from typing import Callable

import dill
import shutil
import sys
import time
import traceback
import multiprocessing

from multiprocessing import Process, Queue

from sessions.dirtools import Dir
from git import Repo
from pathlib import Path

class Logger:
    """Copies terminal output to a file"""
    stderr = False

    def __init__(self, filepath: str,  stderr=False):

        self.stderr = stderr
        if self.stderr:
            self.terminal = sys.stderr
        else:
            self.terminal = sys.stdout
        target = Path(filepath)
        self.log = open(target, "a")

    def write(self, message):
        if self.stderr:
            self.terminal.write('\033[91m' + message + '\033[0m')
        else:
            self.terminal.write(message)
        self.log.write(message)
        self.log.flush()

    def __del__(self):
        try:
            self.stop()
            self.log.close()
        except AttributeError:
            pass

    def stop(self):
        if self.stderr:
            sys.stderr = self.terminal
        else:
            sys.stdout = self.terminal

    def start(self):
        if self.stderr:
            sys.stderr = self
        else:
            sys.stdout = self

    def flush(self):
        self.terminal.flush()

def get_input(valid_inputs=("y", "n")):
    """
    Queries the user for a terminal input which must be one of the valid inputs specified
    :param valid_inputs: A iterable of string the represents valid inputs
    :return:
    """
    res = input().lower()
    while res not in valid_inputs:
        print(f'Valid responses: ({"/".join(valid_inputs)})')
        res = input().lower()
    return res

def load_session(path, use_backup=True):
    """
    Loads a session
    :param path: The session folder (labelled .ses) containng the session.dill and session.back file
    :param use_backup: If True, will attempt to load backup if main fails
    :return:
    """
    target = Path(path) / "session.dill"

    try:
        with open(target, "rb") as fp:
            return dill.load(fp)[0]
    except Exception as e:
        if use_backup:
            print(f"failed to load {target.name}. Reason:")
            target = target.with_suffix(".back")
            time.sleep(0.1)  # Get nicer print statements
            traceback.print_exc()
            time.sleep(0.1)  # Get nicer print statements
            print("Trying to load backup")
            with open(target, "rb") as fp:
                return dill.load(fp)[0]
        else:
            raise e

class Session:
    """A session represents some work that needs to be done and saved, and possibly paused"""
    is_finished = False  # True when the session has finished all the work
    ignore_uncommited_changes_to_main = True
    runtime = datetime.timedelta()
    _repo_dir = Path(os.path.dirname(os.path.abspath(__file__)))

    def __init__(self, worker, name, save_folder=None, repo_dir=None, ignore_file='.ignore', ignore_warnings=True):
        """
        :param worker: A object that implements the work() methods which throws a StopIteration when finished.
        :param name: The name of the session. Unless otherwise specified it will also be the name of the data folder
        :param save_folder: The folder where the data is stored. Default is a folder in the same directory as the
                            main script with same name as the session
        :param repo_dir: The repository in for which the commit state at session runtime is stored if the session is
                            to be restarted at a later time
        :param ignore_file: A file similar to a .gitignore that allows you to specify certain files which can be
                            modified without affecting the result of the worker
        """
        self.ignore_warnings = ignore_warnings
        self.terminate = False
        if repo_dir is not None:
            self._repo_dir = str(repo_dir)
        while ".git" not in os.listdir(self.repo_dir):
            if self.repo_dir == self.repo_dir.parent:
                raise Exception(f"Could not find a .git folder while searching the directory tree")
            self._repo_dir = str(self.repo_dir.parent)
        self.repo = Repo(self.repo_dir)

        self.worker = worker
        self.name = name
        self.ignore_file = ignore_file
        if save_folder is None:
            self._save_folder = f"Experiments/"
        else:
            self._save_folder = str(Path("Experiments") / save_folder)
        os.makedirs(self.save_folder, exist_ok=True)
        self._loggers = [Logger(self.save_folder / "log.txt"), Logger(self.save_folder / "log.txt", stderr=True)]

    @property
    def loggers(self):
        if not hasattr(self, "_loggers"):
            self._loggers = [Logger(self.save_folder / "log.txt"), Logger(self.save_folder / "log.txt", stderr=True)]
        return self._loggers

    @property
    def repo_dir(self):
        return Path(self._repo_dir)

    @property
    def save_folder(self):
        return Path(os.path.dirname(sys.argv[0])) / self._save_folder / (self.name + ".ses")

    def _session_data(self):
        return self.worker, self.repo.head.commit.hexsha, self.is_finished, self.runtime

    def load_results(self):
        """This method is for loading session results after the session has finished"""
        (worker, repo, is_finished) = self.load_data("session")
        commit = repo.index.commit
        if self.repo.index.commit != commit:
            print("Warning: Loaded data belongs to a different commit")

        if not is_finished:
            print("Warning: Loaded data is has not yet finished iterating")
        return worker

    def check_git_status(self):
        """Checks if the there are uncommitted changes to the git head that should be committed before session start"""
        try:
            d = Dir(self.repo.working_dir, exclude_file=self.ignore_file)

            changed_files = [i.a_path for i in self.repo.index.diff(self.repo.head.commit) if
                             not d.is_excluded(Path(self.repo.working_dir) / i.a_path)]
            untracked_files = [f for f in self.repo.untracked_files if
                               not d.is_excluded(Path(self.repo.working_dir) / f)]
            dirty_files = changed_files + untracked_files
            if self.ignore_uncommited_changes_to_main:
                target = sys.argv[0].replace(Path(self.repo_dir).as_posix() + "/", "")
                if target in dirty_files:
                    dirty_files.remove(target)
            if len(dirty_files) > 0:
                print("The following files were untracked or had uncommitted changes:")
                for f in dirty_files:
                    print("- " + f)
                return False
        except Exception as e:
            print("Encountered exception while checking git status")
            traceback.print_exc()
            return False
        return True

    def save_data(self, filename, data, mode="wb", use_backup=True):
        """Saves data in the data folder"""
        with open(Path(self.save_folder) / f"{filename}.dill", mode) as fp:
            dill.dump(data, fp)
        if use_backup:
            with open(Path(self.save_folder) / f"{filename}.back", mode) as fp:
                dill.dump(data, fp)

    def load_data(self, filename, mode="rb", use_backup=True):
        """loads data from the data folder"""
        assert "b" in mode
        try:
            with open(Path(self.save_folder) / f"{filename}.dill", mode) as fp:
                return dill.load(fp)
        except Exception as e:
            if use_backup:
                print(f"failed to load {filename}.pickle. Reason:")
                time.sleep(0.1)  # Get nicer print statements
                traceback.print_exc()
                time.sleep(0.1)  # Get nicer print statements
                print("Trying to load backup")
                with open(Path(self.save_folder) / f"{filename}.back", mode) as fp:
                    return dill.load(fp)
            else:
                raise e

    def start(self, on_load: Callable[['Session'], None] = None):
        """Starts the session. It will guide the user throug a series of questions about choices for the session
        regarding git status and restarting/overwriting previous sessions"""
        if not os.path.exists(self.save_folder):
            os.makedirs(self.save_folder)
        for logger in self.loggers:
            logger.start()
        print(f"--- {self.name} ---")
        status = self.check_git_status()
        if not status:
            if self.ignore_warnings:
                print("Git status is not clean (Ignored)")
            else:
                print("Git status is not clean. Would you still like to continue with the session? (y/n)")
                response = get_input(valid_inputs=("y", "n"))
                if response == "n":
                    return

        if os.path.exists(self.save_folder / "session.dill"):
            print(f"The save folder already exists. (Path: {self.save_folder})")
            if self.ignore_warnings:
                print(f"Loading session {self.name}")
                response = "l"
            else:
                choices = "Choose one:\n- Restart session and overwrite data folder (r)\n- Load folder (l)\n- Exit (q)"
                print(choices)
                response = get_input(valid_inputs=("r", "l", "q"))
            if response == "l":
                data = self.load_data("session")
                if len(data) == 3:
                    (worker, repo, is_finished) = data
                elif len(data) == 4:
                    (worker, repo, is_finished, runtime) = data
                    self.runtime = runtime
                else:
                    raise AssertionError()

                if isinstance(repo, Repo):

                    try:
                        hexsha = repo.head.commit.hexsha
                    except ValueError:
                        print("(Probably) got error when comparing loaded head with current. This most likely is an "
                              "issue of data stored with an old version of session when loading data on a different PC")
                        traceback.print_exc()
                        hexsha = None
                else:
                    assert isinstance(repo, str)
                    hexsha = repo
                if self.repo.head.commit.hexsha != hexsha:
                    print("WARNING: The loaded session belonged to a different commit and cannot be loaded")
                    print(f"Consider rerunning script after checking out commit({hexsha})")
                    if not self.ignore_warnings:
                        print("Continue? (Y/N)")
                        response = get_input(valid_inputs=("y", "n",))
                        if response == "n":
                            return
                self.worker = worker
                if on_load is not None:
                    print(f"Calling on_load method: {on_load.__name__}")
                    on_load(self)
                self._run()
                return
            elif response == "r":
                print("Confirm (y/n)")
                if get_input(("y", "n")) == "y":
                    shutil.rmtree(self.save_folder)
                    os.makedirs(self.save_folder)
                else:
                    return
            else:
                return
        self.save_data("session", self._session_data())
        shutil.copyfile(Path(sys.argv[0]), Path(self.save_folder) / "script_copy.py")
        print(f"Starting session: {self.name}")
        self._run()

    def _run(self):
        """Don't call explicitly. Instead use start()
        Lets the worker (continue) work until interrupted or finished
        """
        while True:
            try:
                starttime = datetime.datetime.now()
                self.worker.iterate()
                self.runtime += (datetime.datetime.now() - starttime)
                self.save_data("session", self._session_data())
                if self.terminate:
                    print("Session terminated")
                    return
            except StopIteration:
                break
        self.is_finished = True
        self.save_data("session", self._session_data())
        print(f"Session done ({self.name}) in total time: {self.runtime}")
        for logger in self.loggers:
            logger.stop()

    # serialization
    def __getstate__(self):
        state = self.__dict__.copy()
        if "_loggers" in state:
            del state['_loggers']
        return state

    # # serialization
    # def __setstate__(self, state):
    #     for s in ["logger", "_logger", "loggers", "_logger"]:
    #         if s in state:
    #             del state[s]
    #     self.__dict__ = state

class MultiSession(Session):
    """Works like a session except it accepts a list of workers and executes them sequentially"""

    def __init__(self, workers, name, save_folder=None, repo_dir=None, ignore_file='.ignore',
                 ignore_warnings=True, parallel_execution=False):
        self.workers = workers
        self.current_worker = 0
        super().__init__(self, name, save_folder=save_folder, repo_dir=repo_dir, ignore_file=ignore_file,
                         ignore_warnings=ignore_warnings)
        self.parallel_execution = parallel_execution  # TODO: This implementation is quick and dirty hand has potential problems for speical uses
        self.completed = [False] * len(self.worker.workers)
        self.errors = [False] * len(self.worker.workers)

    def _work(self):
        try:
            self.worker.workers[self.current_worker].iterate()
        except StopIteration:
            print(f"Finished worker ({self.current_worker})")
            self.completed[self.current_worker] = True
        except Exception as e:
            print(f"Error in worker ({self.current_worker})")
            traceback.print_exc()
            self.completed[self.current_worker] = True
            self.errors[self.current_worker] = True
        return self.completed[self.current_worker]

    def iterate(self):
        if not all(self.completed):
            if self.parallel_execution:
                for _ in range(len(self.worker.workers)):
                    if not self.completed[self.current_worker]:
                        break
                    self.current_worker = (self.current_worker + 1) % len(self.worker.workers)
                self._work()
                self.current_worker = (self.current_worker + 1) % len(self.worker.workers)
            else:
                done = self._work()
                if done:
                    self.current_worker = (self.current_worker + 1) % len(self.worker.workers)

        else:
            if any(self.errors):
                error_idx = [i for i, err in enumerate(self.errors) if err]
                print(f"WARNING: Unhandled exceptions occurred in thread {error_idx}")
            raise StopIteration()

class MultiThreadedSession(Session):
    """Works like a MultiSession except all workers work concurrently"""

    def __init__(self, workers, name, save_folder=None, repo_dir=None, ignore_file='.ignore', ignore_warnings=True,
                 thread_count: int = None):
        self.workers = workers

        self.status_done = [False] * len(workers)
        self.status_error = [False] * len(workers)
        self.status_working = [False] * len(workers)
        super().__init__(None, name, save_folder=save_folder, repo_dir=repo_dir, ignore_file=ignore_file,
                         ignore_warnings=ignore_warnings)
        self.worker = self
        self.i = 0
        self._thread_count = thread_count
        # raise NotImplementedError("Does not currently work")

    @property
    def thread_count(self):
        return min(len(self.workers),
                   self._thread_count if self._thread_count is not None else multiprocessing.cpu_count())

    def _get_next_worker(self):
        i = int(self.i)
        self.i = (self.i + 1) % len(self.workers)
        for _ in range(len(self.workers)):
            if not self.status_done[i] and not self.status_working[i]:
                return i, self.workers[i]
            i = (i + 1) % len(self.workers)

    def _run(self):
        """Don't call explicitly. Instead use start()
                Lets the worker (continue) work until interrupted or finished
                """
        queue = Queue()

        for i in range(self.thread_count):
            res = self._get_next_worker()
            if res:
                i, worker = res
                self.status_working[i] = True
                p = Process(target=self._process_worker, args=(i, worker, queue))
                p.start()

        while any(self.status_working):
            i, worker, done, error, runtime = queue.get()

            self.workers[i] = worker
            self.status_error[i] = error
            self.status_done[i] = done
            self.status_working[i] = False
            self.runtime += runtime

            res = self._get_next_worker()
            if res:
                i, worker = res
                self.status_working[i] = True
                p = Process(target=self._process_worker, args=(i, worker, queue))
                p.start()
        self.is_finished = True
        self.save_data("session", self._session_data())

        print(f"Session done ({self.name}) in total time: {self.runtime}")
        for logger in self.loggers:
            logger.stop()

    def _process_worker(self, worker_id: int, worker, queue: Queue):
        done, error = False, False
        starttime = datetime.datetime.now()
        try:
            worker.iterate()
        except StopIteration:
            print(f"Finished worker ({worker_id})")
            done = True
        except Exception as e:
            done, error = True, True
            print(f"Error in worker ({worker_id})")
            traceback.print_exc()
        runtime = (datetime.datetime.now() - starttime)
        queue.put((worker_id, worker, done, error, runtime))


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from torchvision import transforms
from torchvision.datasets import MNIST
import torchvision.models as models

import numpy as np
import matplotlib.pyplot as plt

from advertorch.attacks import GradientSignAttack as FGSM

class MnistModel(nn.Module):
    def __init__(self):
        super(MnistModel, self).__init__()
        # mnist의 경우 28*28의 흑백이미지(input channel=1)이다.
        self.conv1 = nn.Conv2d(1, 32, kernel_size = 5, padding=2)
        # feature map의 크기는 14*14가 된다
        # 첫번재 convolution layer에서 나온 output channel이 32이므로 2번째 input도 32
        self.conv2 = nn.Conv2d(32, 64, kernel_size = 5, padding=2)
        # feature map의 크기는 7*7이 된다
        # fc -> fully connected, fc는 모든 weight를 고려해서 만들기 때문에 cnn에서는 locally connected를 이용하여 만든다.
        # nn.Linear에서는 conv를 거친 feature map을 1차원으로 전부 바꿔서 input을 한다. 이게 64*7*7
        self.fc1 = nn.Linear(64*7*7, 1024)
        self.fc2 = nn.Linear(1024, 10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), 2)
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(-1, 64*7*7) # linear에 들어갈 수 있도록 reshape
        x = F.relu(self.fc1(x)) # fully connected에 relu 적용
        x = F.dropout(x, training=self.training) # 가중치 감소만으로는 overfit을 해결하기가 어려움, 그래서 뉴런의 연결을 임의로 삭제
        x = self.fc2(x)
        return F.log_softmax(x)

def advtrain(model, device, train_loader, optimizer, epoch, log_interval):
    model.train()
    avg_loss = 0
    # in training loop:

    adversary = FGSM(model, loss_fn=nn.NLLLoss(reduction='sum'),
                    eps=0.3, clip_min=0., clip_max=1., targeted=False)

    for batch_idx, (data, target) in enumerate(train_loader):
        # gpu나 cpu로 데이터를 옮김
        data, target = data.to(device), target.to(device)
        data = adversary.perturb(data, target)
        # gradient descent전에 gradient buffer를 0으로 초기화 하는 역할을 한다
        optimizer.zero_grad()
        output = model(data)
        # negative log-likelihood: nll loss, 딥러닝 모델의 손실함수이다
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step() # Does the update based on the current gradient stored in grad
        # 여기서 기존에 저장되어있던 gradient를 기반으로 update 하기 때문에 위의 초기화가 필요함
        avg_loss+=F.nll_loss(output, target, reduction='sum').item()

        if batch_idx % log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))
    avg_loss/=len(train_loader.dataset)
    return avg_loss

train_batch_size=64
test_batch_size=1
epochs=30
lr = 0.01
momentum = 0.5
seed = 1
log_interval = 1000

is_cuda = torch.cuda.is_available()
device = torch.device('cuda' if is_cuda else 'cpu')

pretrained_model = './data/mnist_model.pth'
model = MnistModel().to(device)
model.load_state_dict(torch.load(pretrained_model, map_location='cpu'))

# momentum: 기울기에서 속도의 개념을 추가, 기울기 업데이트시 폭을 조절한다
# lr: learning rate
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)

# mean=0.5 std=1.0으로 Normalize한다
mnist_transform = transforms.Compose([
    transforms.ToTensor(),
    # transforms.Normalize((0.5,), (1.0,))
])

download_path = './data'
train_dataset = MNIST(download_path, transform=mnist_transform, train=True, download=True)
test_dataset = MNIST(download_path, transform=mnist_transform, train=False, download=True)

train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=train_batch_size,
                                           shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                          batch_size=test_batch_size,
                                          shuffle=True)

train_losses = []
test_losses = []
accuracy_list = []

for epoch in range(1, epochs + 1):
    trn_loss = advtrain(model, device, train_loader, optimizer, epoch, log_interval)
    #test_loss,accuracy = test(model, device, test_loader)
    train_losses.append(trn_loss)
    #test_losses.append(test_loss)
    #accuracy_list.append(accuracy)

PATH = './data/mnist_fgsm_model.pth'
torch.save(model.state_dict(), PATH)


#importing the openCV and numpy packages
import cv2
import numpy as np

#creating a balck image of 300x300
canvas = np.zeros((300, 300, 3), dtype = "uint8")

#creating a green line from 0,0 to 300,300 and another line with thickness of 3 pixel from 300,0 to 0,300
#syntax: cv2.line(image,starting_point,ending_point,color)
green = (0, 255, 0)
cv2.line(canvas, (0, 0), (300, 300), green)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)

#syntax: cv2.line(image,starting_point,ending_point,color,thickness)
red = (0, 0, 255)
cv2.line(canvas, (300, 0), (0, 300), red, 3)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)

#green rectangle
cv2.rectangle(canvas, (10, 10), (60, 60), green)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)

#thick red rectangle
cv2.rectangle(canvas, (50, 200), (200, 225), red, 5)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)

#thcikness as -1 will cause color fill
blue = (255, 0, 0)
cv2.rectangle(canvas, (200, 50), (225, 125), blue, -1)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)

#creating circles
white = (255, 255, 255)
centerX = 125
centerY = 125
r=50
cv2.circle(canvas, (centerX, centerY), r, white,2)
cv2.imshow("Canvas", canvas)
cv2.waitKey(0)
cv2.destroyAllWindows()


import os
import numpy as np
import cv2
import torch
import torch.nn as nn
import torchvision as tv

import network
import dataset

# ----------------------------------------
#                 Network
# ----------------------------------------
def create_generator(opt):
    # Initialize the networks
    generator = network.GatedGenerator(opt)
    print('Generator is created!')
    if opt.load_name:
        generator = load_dict(generator, opt.load_name)
    else:
        # Init the networks
        network.weights_init(generator, init_type = opt.init_type, init_gain = opt.init_gain)
        print('Initialize generator with %s type' % opt.init_type)
    return generator

def create_discriminator(opt):
    # Initialize the networks
    discriminator = network.PatchDiscriminator(opt)
    print('Discriminator is created!')
    # Init the networks
    network.weights_init(discriminator, init_type = opt.init_type, init_gain = opt.init_gain)
    print('Initialize discriminator with %s type' % opt.init_type)
    return discriminator

def create_perceptualnet():
    # Get the first 15 layers of vgg16, which is conv3_3
    perceptualnet = network.PerceptualNet()
    # Pre-trained VGG-16
    vgg16 = torch.load('./vgg16_pretrained.pth')
    load_dict(perceptualnet, vgg16)
    # It does not gradient
    for param in perceptualnet.parameters():
        param.requires_grad = False
    print('Perceptual network is created!')
    return perceptualnet

def load_dict(process_net, pretrained_net):
    # Get the dict from pre-trained network
    pretrained_dict = pretrained_net
    # Get the dict from processing network
    process_dict = process_net.state_dict()
    # Delete the extra keys of pretrained_dict that do not belong to process_dict
    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in process_dict}
    # Update process_dict using pretrained_dict
    process_dict.update(pretrained_dict)
    # Load the updated dict to processing network
    process_net.load_state_dict(process_dict)
    return process_net

# ----------------------------------------
#             PATH processing
# ----------------------------------------
def text_readlines(filename):
    # Try to read a txt file and return a list.Return [] if there was a mistake.
    try:
        file = open(filename, 'r')
    except IOError:
        error = []
        return error
    content = file.readlines()
    # This for loop deletes the EOF (like \n)
    for i in range(len(content)):
        content[i] = content[i][:len(content[i])-1]
    file.close()
    return content

def savetxt(name, loss_log):
    np_loss_log = np.array(loss_log)
    np.savetxt(name, np_loss_log)

def get_files(path):
    # read a folder, return the complete path
    ret = []
    for root, dirs, files in os.walk(path):
        for filespath in files:
            ret.append(os.path.join(root, filespath))
    return ret

def get_names(path):
    # read a folder, return the image name
    ret = []
    for root, dirs, files in os.walk(path):
        for filespath in files:
            ret.append(filespath)
    return ret

def text_save(content, filename, mode = 'a'):
    # save a list to a txt
    # Try to save a list variable in txt file.
    file = open(filename, mode)
    for i in range(len(content)):
        file.write(str(content[i]) + '\n')
    file.close()

def check_path(path):
    if not os.path.exists(path):
        os.makedirs(path)

# ----------------------------------------
#    Validation and Sample at training
# ----------------------------------------
def save_sample_png(sample_folder, sample_name, img_list, name_list, pixel_max_cnt = 255):
    # Save image one-by-one
    for i in range(len(img_list)):
        img = img_list[i]
        # Recover normalization: * 255 because last layer is sigmoid activated
        img = img * 255
        # Process img_copy and do not destroy the data of img
        img_copy = img.clone().data.permute(0, 2, 3, 1)[0, :, :, :].cpu().numpy()
        img_copy = np.clip(img_copy, 0, pixel_max_cnt)
        img_copy = img_copy.astype(np.uint8)
        # Save to certain path
        save_img_name = sample_name + '_' + name_list[i] + '.png'
        save_img_path = os.path.join(sample_folder, save_img_name)
        cv2.imwrite(save_img_path, img_copy)

def psnr(pred, target, pixel_max_cnt = 255):
    mse = torch.mul(target - pred, target - pred)
    rmse_avg = (torch.mean(mse).item()) ** 0.5
    p = 20 * np.log10(pixel_max_cnt / rmse_avg)
    return p

def grey_psnr(pred, target, pixel_max_cnt = 255):
    pred = torch.sum(pred, dim = 0)
    target = torch.sum(target, dim = 0)
    mse = torch.mul(target - pred, target - pred)
    rmse_avg = (torch.mean(mse).item()) ** 0.5
    p = 20 * np.log10(pixel_max_cnt * 3 / rmse_avg)
    return p

def ssim(pred, target):
    pred = pred.clone().data.permute(0, 2, 3, 1).cpu().numpy()
    target = target.clone().data.permute(0, 2, 3, 1).cpu().numpy()
    target = target[0]
    pred = pred[0]
    ssim = skimage.measure.compare_ssim(target, pred, multichannel = True)
    return ssim


from pylab import *
import scipy
from scipy.signal import lfilter
def dirac(n):
    d=zeros(n)
    d[0]=1
    return d
a1=-1.76
a2=0.8
N=100
x=dirac(N)
y=lfilter([1,1/2],[1, a1 , a2],x)
stem(y)


import llvmlite.ir as ll
import llvmlite.binding as llvm

import logging
import time

from . import tp
from . import ir
from . import exc

class CGEnv(object):
    module = None
    builder = None

class Program(object):
    def __init__(self, module):
        llvm.initialize()
        llvm.initialize_native_target()
        llvm.initialize_native_asmprinter()

        self.module = module
        self.module.translate()

        self.cge = CGEnv()
        self.cge.module = module

        self.llvm = self.makeStub()

        for _, func in self.module.namestore.all(ir.Function):
            self.blockAndCode(func)

        self.target_machine = llvm.Target.from_default_triple().create_target_machine()

        logging.debug("Verifying... ")
        self._llmod = None

    def llmod(self):
        if not self._llmod:
            self._llmod = llvm.parse_assembly(str(self.module.llvm))
        return self._llmod

    def blockAndCode(self, impl):
        func = impl.llvm
        # create blocks
        bb = func.append_basic_block("entry")

        for bc in impl.bytecodes:
            if bc.discard:
                impl.remove(bc)
                impl.log.debug("BLOCK skipped {0}".format(bc))
                continue

            newblock = ''
            if bc in impl.incoming_jumps:
                assert not bc.block
                bc.block = func.append_basic_block(str(bc.loc))
                bb = bc.block
                newblock = ' NEW BLOCK (' + str(bc.loc) + ')'
            else:
                bc.block = bb
            impl.log.debug("BLOCK'D {0}{1}".format(bc, newblock))

        for ext_module in self.module.getExternalModules():
            ext_module.translate(self.module.llvm)

        impl.log.debug("Printing all bytecodes:")
        impl.bytecodes.printAll(impl.log)

        impl.log.debug("Emitting code:")
        bb = None
        cge = self.cge
        for bc in impl.bytecodes:
            try:
                if bb != bc.block:
                    # new basic block, use a new builder
                    cge.builder = ll.IRBuilder(bc.block)

                if bc.reachable:
                    bc.translate(cge)
                    impl.log.debug("TRANS'D {0}".format(bc.locStr()))
                else:
                    # eliminate unreachable code, which may occur in the middle of a function
                    impl.log.debug("UNREACH {}".format(bc.locStr()))
            except exc.StellaException as e:
                e.addDebug(bc.debuginfo)
                raise

    def makeStub(self):
        impl = self.module.entry
        func_tp = ll.FunctionType(impl.result.type.llvmType(self.module), [])
        func = ll.Function(self.module.llvm, func_tp, name=str(impl.function)+'__stub__')
        bb = func.append_basic_block("entry")
        builder = ll.IRBuilder(bb)
        self.cge.builder = builder

        for name, var in self.module.namestore.all(ir.GlobalVariable):
            var.translate(self.cge)

        llvm_args = [arg.translate(self.cge) for arg in self.module.entry_args]

        call = builder.call(impl.llvm, llvm_args)

        if impl.result.type is tp.Void:
            builder.ret_void()
        else:
            builder.ret(call)
        return func

    def elapsed(self):
        if self.start is None or self.end is None:
            return None
        return self.end - self.start

    def optimize(self, opt):
        if opt is not None:
            logging.warn("Running optimizations level {0}... ".format(opt))

            # TODO was build_pass_managers(tm, opt=opt, loop_vectorize=True, fpm=False)
            pmb = llvm.create_pass_manager_builder()
            pmb.opt_level = opt
            pm = llvm.create_module_pass_manager()
            pmb.populate(pm)
            pm.run(self.llmod())

    def destruct(self):
        self.module.destruct()
        del self.module

    def __del__(self):
        logging.debug("DEL  {}: {}".format(repr(self), hasattr(self, 'module')))

    def run(self, stats):
        logging.debug("Preparing execution...")

        import ctypes
        import llvmlite
        import os

        _lib_dir = os.path.dirname(llvm.ffi.__file__)
        clib = ctypes.CDLL(os.path.join(_lib_dir, llvmlite.utils.get_library_name()))
        # Direct access as below mangles the name
        # f = clib.__powidf2
        f = getattr(clib, '__powidf2')
        llvm.add_symbol('__powidf2', ctypes.cast(f, ctypes.c_void_p).value)

        with llvm.create_mcjit_compiler(self.llmod(), self.target_machine) as ee:
            ee.finalize_object()

            entry = self.module.entry
            ret_type = entry.result.type

            logging.info("running {0}{1}".format(entry,
                                                 list(zip(entry.type_.arg_types,
                                                          self.module.entry_args))))

            entry_ptr = ee.get_pointer_to_global(self.llmod().get_function(self.llvm.name))
            ret_ctype = entry.result.type.Ctype()
            if ret_type.on_heap:
                ret_ctype = ctypes.POINTER(ret_ctype)
            cfunc = ctypes.CFUNCTYPE(ret_ctype)(entry_ptr)

            time_start = time.time()
            retval = cfunc()
            stats['elapsed'] = time.time() - time_start

        for arg in self.module.entry_args:
            arg.ctype2Python(self.cge)  # may be a no-op if not necessary

        retval = ret_type.unpack(retval)

        logging.debug("Returning...")
        self.destruct()

        return retval

    def getAssembly(self):
        return self.target_machine.emit_assembly(self.llmod())

    def getLlvmIR(self):
        ret = self.module.getLlvmIR()

        logging.debug("Returning...")
        self.destruct()

        return ret


from parse_page import parse_page
from selenium import webdriver

browser = webdriver.Firefox()
cnt = 1

while True:
    next_page = parse_page(
        f'https://www.kijiji.ca/b-apartments-condos/city-of-toronto/page-{cnt}/c37l1700273',
        browser)
    if not next_page:
        break
    print(f'End of page {cnt}')
    cnt += 1

browser.close()


from base_test import BaseTestApi
import requests
import xmltodict

class TestUpdateIssue(BaseTestApi):

    def setUp(self):
        super(TestUpdateIssue, self).setUp()
        self.url = self.base_url + '/issue/'

        self.updateData = {
            'summary': 'updated_test_summary',
            'description': 'updated_test_description'
        }

    def test_update_issue(self):
        issue_id = self.create_issue()
        url = self.url + issue_id

        r = requests.post(url, self.updateData, cookies=self.cookies)

        self.assertEquals(r.status_code, 200)

        response = requests.get(url, cookies=self.cookies)
        response_dict = xmltodict.parse(response.text)

        self.assertEquals(response_dict['issue']['field'][2]['value'], self.updateData['summary'])
        self.assertEquals(response_dict['issue']['field'][3]['value'], self.updateData['description'])

        #remove created issue after all checks
        self.delete_issue(issue_id)

    def test_update_nonexisting_issue(self):
        url = self.base_url + 'ZZZ'

        r = requests.post(url, self.updateData, cookies=self.cookies)

        self.assertEquals(r.status_code, 404)


# Author: HZHtat
# Date: Jan-2020
# Version: 0.8

import smtplib
from email.message import EmailMessage

def alerter(emailpackage, mode, to, body):
    '''
    emailpackage is a list:
    [sender, [receiver(s),], error_subject, success_subject, mailsvr]
    Pass through a list of following structure. e.g.
    ([
    'from',
    {'prim': x, 'sec': y, 'ter': z},
    {'err': ..., 'success': ..., 'info': ...},
    mailsvr
    ],
    'mode',
    'to',
    'body')
    Arg exception is either True or False
    '''
    msg = EmailMessage()
    msg['From'] = emailpackage[0]

    if to == 'prim':
        msg['To'] = emailpackage[1]['prim']
    elif to == 'sec':
        msg['To'] = emailpackage[1]['sec']
    elif to == 'ter':  # tertiary
        msg['To'] = emailpackage[1]['ter']

    msg.set_content(body)

    # subject lines
    if mode == 'err':
        msg['Subject'] = emailpackage[2]['err']
    elif mode == 'success':
        msg['Subject'] = emailpackage[2]['success']
    elif mode == 'info':
        msg['Subject'] = emailpackage[2]['info']

    s = smtplib.SMTP(emailpackage[3])  # mailserver
    s.send_message(msg)
    s.quit()


#!/usr/bin/env python

import netCDF4 as nc4

import matplotlib as mpl
import matplotlib.pyplot as plt

import datetime
import numpy as np

import glob
import sys, cftime

#ilamb_root="/qfs/people/xumi699/scratch/proc_ilamb4xj/ilamb_wks/_build/EcosystemandCarbonCycle/GlobalNetEcosystemCarbonBalance"
ilamb_root="/qfs/people/xumi699/scratch/proc_ilamb4xj/ilamb_wks/_build_useneedirectly/"

ncfiles = glob.glob(ilamb_root + "/EcosystemandCarbonCycle/GlobalNetEcosystemCarbonBalance/GCP/*.nc")

print (ncfiles)

with nc4.Dataset("GCP_Benchmark.nc", "r") as newncf:

    xxnew = newncf.groups['MeanState'].variables['time_'][:] / 365. + 1851
    yynew = newncf.groups['MeanState'].variables['accumulate_of_nbp_over_global'][:] * (-1.)
    ybnew = newncf.groups['MeanState'].variables['accumulate_of_nbp_over_global_bnds'][:,0] * (-1.)
    ytnew = newncf.groups['MeanState'].variables['accumulate_of_nbp_over_global_bnds'][:,1] * (-1.)

fig, ax = plt.subplots()

for ncfile in ncfiles:
    print (ncfile)
    with nc4.Dataset(ncfile, 'r') as ncf:

         xx = ncf.groups['MeanState'].variables['time_'][:] / 365. + 1851
         yy = ncf.groups['MeanState'].variables['accumulate_of_nbp_over_global'][:] * (-1.)

         if 'Benchmark' in ncfile:
             ax.fill_between(xxnew, ybnew, ytnew, lw=0, color = 'k', alpha = 0.25)
             #ax.plot(xxnew, yynew, '--', color='black', label='Benchmark New')
             ax.plot(xx, yy, '-', color='black', label='Benchmark')

         elif 'CNP' in ncfile:
             ax.plot(xx, yy, '-', color='blue', label='ELMv1-CNP')
         elif 'CNonly' in ncfile:
             ax.plot(xx, yy, '-', color='red', label='ELMv1-CN')

         ax.set_xlabel("Year", fontsize="large", fontweight="bold")
         ax.set_xlim(1960, 2010)
         ax.set_ylabel("Accumulative land carbon sink (PgC)", fontsize="large", fontweight="bold")
         ax.set_ylim(-30, 80)

         ax.tick_params(labelsize='large', width=1)
         ax.legend(bbox_to_anchor=(0.05, 1), loc='upper left', borderaxespad=0.)

ax.hlines(0, 1850, 2010, colors='grey', linestyles='--')

plt.savefig("ngepbal_gcp_withuncertainty.png", dpi=300)


"""Platform for Wattio integration testing."""
import logging

from homeassistant.const import (
    ATTR_BATTERY_LEVEL
)
from homeassistant.helpers.entity import Entity

from . import WattioDevice

from .const import DOMAIN, ICON, MEASUREMENT, SENSORS

_LOGGER = logging.getLogger(__name__)

async def async_setup_platform(hass, config, async_add_entities, discovery_info=None):
    """Wattio Sensor setup platform."""
    _LOGGER.debug("Wattio Sensor component running ...")
    if discovery_info is None:
        _LOGGER.error("No Sensor(s) discovered")
        return
    devices = []
    for device in hass.data[DOMAIN]["devices"]:
        if device["type"] in SENSORS:
            if device["type"] == "bat":
                channel = device["channel"]
            else:
                channel = None
            devices.append(
                WattioSensor(
                    device["name"],
                    device["type"],
                    MEASUREMENT[device["type"]],
                    ICON[device["type"]],
                    device["ieee"],
                    channel,
                )
            )
            _LOGGER.debug("Adding device: %s", device["name"])

    async_add_entities(devices)

class WattioSensor(WattioDevice, Entity):
    """Representation of Sensor."""

    # pylint: disable=too-many-instance-attributes

    def __init__(self, name, devtype, measurement, icon, ieee, channel=None):
        """Initialize the sensor."""
        self._pre = "s_"
        self._name = name
        self._state = None
        self._measurement = measurement
        self._icon = icon
        self._ieee = ieee
        self._devtype = devtype
        self._channel = None
        self._battery = None
        self._data = None
        self._available = 0
        if channel is not None:
            self._channel = channel - 1

    @property
    def name(self):
        """Return the name of the sensor."""
        return self._name

    @property
    def available(self):
        """Return availability."""
        _LOGGER.debug("Device %s - availability: %s", self._name, self._available)
        return True if self._available == 1 else False

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return self._measurement

    @property
    def icon(self):
        """Return the image of the sensor."""
        return self._icon

    @property
    def state(self):
        """Return sensor state."""
        return self._state

    @property
    def device_state_attributes(self):
        """Return the state attributes of this device."""
        attr = {}
        if self._battery is not None:
            attr[ATTR_BATTERY_LEVEL] = self.get_battery_level()
        return attr

    def get_battery_level(self):
        """Return device battery level."""
        if self._battery is not None:
            battery_level = round((self._battery * 100) / 4)
            return battery_level
        return False

    async def async_update(self):
        """Update sensor data."""
        self._data = self.hass.data[DOMAIN]["data"]
        _LOGGER.debug("ACTUALIZANDO SENSOR %s - %s", self._name, self._ieee)
        if self._data is not None:
            self._available = 0
            for device in self._data:
                if device["ieee"] == self._ieee:
                    self._available = 1
                    if self._channel is not None and self._devtype == "bat":
                        sensorvalue = device["status"]["consumption"][self._channel]
                    elif self._devtype == "therm":
                        sensorvalue = device["status"]["current"]
                    elif self._devtype == "motion":
                        sensorvalue = device["status"]["temperature"]
                        self._battery = device["status"]["battery"]
                    elif self._devtype == "pod":
                        sensorvalue = device["status"]["consumption"]
                    else:
                        sensorvalue = None
                    self._state = sensorvalue
                    break
            _LOGGER.debug("Valor: %s", self._state)
            return self._state
        return False


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse
from rest_framework.decorators import api_view, permission_classes
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated, IsAdminUser
from .serializers import MovieSerializer, MovieListSerializer, GenreSerializer, ReviewSerializer, ReviewListSerializer
from .models import Movie, Genre, Review

# Create your views here.
@api_view(['GET'])
def genre_list(request):
    genres = Genre.objects.all()
    serializer = GenreSerializer(genres, many=True)
    return Response(serializer.data)

@api_view(['GET'])
def movie_list(request):
    movies = Movie.objects.all()
    serializer = MovieListSerializer(movies, many=True)
    return Response(serializer.data)

@api_view(['POST'])
@permission_classes([IsAdminUser])
def movie_create(request):
    serializer = MovieSerializer(data=request.data)
    if serializer.is_valid(raise_exception=True):
        serializer.save()
        return Response(serializer.data)

@api_view(['GET'])
def movie_detail(request, movie_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    if request.method == 'GET':
        serializer = MovieSerializer(movie)
        return Response(serializer.data)

@api_view(['PUT', 'DELETE'])
@permission_classes([IsAdminUser])
def movie_update(request, movie_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    if request.method == 'PUT':
        serializer = MovieSerializer(data=request.data, instance=movie)
        if serializer.is_valid(raise_exception=True):
            serializer.save()
            return Response(serializer.data)
    else:
        movie.delete()
        return Response({'message': 'Movie has been deleted!'})

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def user_reviews(request, user_name):
    if request.user.username == user_name:
        reviews = Review.objects.filter(user=request.user.id)
        serializer = ReviewListSerializer(reviews, many=True)
        return Response(serializer.data)
    else:
        return Response({'message': 'You do not have permission!'})

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def recommend_movie(request, user_name):
    if request.user.username == user_name:
        # request user's reviews
        reviews = Review.objects.filter(user=request.user.id)
        # number of reviews less than 3
        if len(reviews) < 3:
            # random
            movies = Movie.objects.order_by("?")[:12]
            serializer = MovieListSerializer(movies, many=True)
            return Response(serializer.data)
        else:
            movie_dict = {}
            for review in reviews:
                movies = Movie.objects.filter(pk=review.movie_id)
                serializer = MovieListSerializer(movies, many=True)
                # genre count
                for movie in serializer.data:
                    for genre in movie['genres']:
                        if genre not in movie_dict:
                            movie_dict[genre] = 1
                        else:
                            movie_dict[genre] += 1
            # sort genre
            movies_arr = sorted(movie_dict.items(), key= lambda x: -x[1])
            # the most seleted genre
            genre = Genre.objects.filter(pk=movies_arr[0][0])
            genre_serializer = GenreSerializer(genre, many=True)
            # Search for a movie that corresponds to the genre
            recommend = genre_serializer.data[0]['movie_genres']
            recommend = recommend[:12]
            movies = Movie.objects.filter(id__in=recommend)
            serializer = MovieListSerializer(movies, many=True)
            return Response(serializer.data)
    else:
        return Response({'message': 'You do not have permission!'})

@api_view(['GET'])
def review_list(request, movie_pk):
    reviews = Review.objects.filter(movie=movie_pk)
    serializer = ReviewListSerializer(reviews, many=True)
    return Response(serializer.data)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def review_create(request, movie_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    serializer = ReviewSerializer(data=request.data)
    if serializer.is_valid(raise_exception=True):
        serializer.save(user=request.user, movie=movie)
        return Response(serializer.data)

@api_view(['PUT', 'DELETE'])
@permission_classes([IsAuthenticated])
def review_update(request, movie_pk, review_pk):
    review = get_object_or_404(Review, pk=review_pk)
    if request.user.id == review.user.id:
        if request.method == 'PUT':
            serializer = ReviewSerializer(data=request.data, instance=review)
            if serializer.is_valid(raise_exception=True):
                serializer.save()
                return Response(serializer.data)
        else:
            review.delete()
            return Response({'message': 'Review has been deleted!'})
    else:
        return Response({'message': 'Author only'})


import cv2
import matplotlib.pyplot as plt
import numpy as np
import pyzbar.pyzbar as pyzbar

width, height = 0, 0

def decode(img):
    # if height < width, then rotate the image
    img = img if img.shape[0] > img.shape[1] else np.rot90(img)

    global height, width
    width, height = img.shape[1], img.shape[0]

    roi_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    roi_map = cv2.convertScaleAbs(roi_gray, alpha=1, beta=0)  # 將灰階轉換成黑白圖，增強對比度

    _, roi_thresh = cv2.threshold(
        roi_map, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)  # 大津法閾值化

    # 一些可能用的功能
    # roi_thresh = cv2.bitwise_not(roi_thresh)  # 反轉圖像

    roi_thresh = cv2.inpaint(roi_thresh, np.zeros(
        roi_thresh.shape, dtype=np.uint8), 3, cv2.INPAINT_TELEA)  # 填充黑色區域

    roi_thresh = fill_color(roi_thresh, 255)
    # Border
    borders = 250
    canvas = np.zeros((height+2*borders, width+2*borders), dtype=np.uint8)
    canvas[:] = 255

    canvas[borders:-borders, borders:-borders] = roi_thresh

    # cv2.imshow('canvas', canvas)
    # cv2.waitKey(0)

    return process1(canvas)

def process1(img):
    img_cp = img.copy()
    roi_blur = cv2.GaussianBlur(img_cp, (5, 5), 0)
    roi_edge = cv2.Canny(roi_blur, 50, 100, apertureSize=3)

    contours, _ = cv2.findContours(
        roi_edge, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)  # 找輪廓

    # 依照凸包面積大小來排序
    contours = sorted(contours, key=lambda c: cv2.contourArea(
        cv2.convexHull(c)), reverse=True)

    contours = [contours[0]]  # 只留下最大的輪廓

    # # show image
    # cp = roi_edge.copy()
    # convex = [cv2.convexHull(contours) for contours in contours]  # 找輪廓的凸包
    # cv2.polylines(cp, convex, True, 255, 3)
    # cv2.namedWindow("convex", cv2.WINDOW_NORMAL)
    # cv2.imshow("convex", cp)

    # 畫出凸包
    convex = draw_approx_hull_polygon(img_cp, contours)  # 繪製凸包
    # 取得角點
    pts = getcorner(convex)

    # pt0--------pt1
    # |           |
    # pt3--------pt2
    h, w = pts[3][1]-pts[0][1], pts[2][0]-pts[3][0]

    img = Perspective(img, pts, w, h)

    return process2(img)

def process2(img):
    # 上一步已經取得了QR code的4個角點，並根據角點的位置
    # 把圖片不需要的部分裁切掉了

    # 去除黑框

    dst = fill_color(img, 255)

    cv2.inpaint(dst, np.ones(dst.shape, dtype=np.uint8),
                3, cv2.INPAINT_TELEA, dst)
    # cv2.imshow('dst', dst)
    dst = np.where(dst < 120, 0, 255).astype(np.uint8)
    # 我們現在要找的東西是這樣的:

    # w w w w w w w w #   <- start #1
    # ............... #
    #                 #
    #     上半部
    #                 #
    # ............... #
    # w w w w w w w w #   <- end #1
    # w w w w w w w w #   <- start #2
    # *************** #
    #
    #     下半部
    #
    # *************** #
    # w w w w w w w w #   <- end #2

    # wt_row = np.where((dst == np.full(w, 255, dtype=np.uint8)).all(axis=1))[0]
    # if len(wt_row) < 3:
    #     return None

    # 先找到上半部的起始和結束點
    h, w = np.int0(dst.shape)

    res = np.where((dst == np.full(w, 255, dtype=np.uint8)).all(axis=1))[0]
    y1 = res[np.where((res < h/10))][-1]
    y2 = res[np.where((res > h/3) & (res < h*2/3))][0]  # <- start #2
    y3 = res[np.where((res > h/3) & (res < h*2/3))][-1]  # <- end #1, start #2
    y4 = res[np.where((res > h*9/10))][0]   # <- end #2

    half_1 = dst[y1:y2, :]
    half_2 = dst[y3:y4, :]

    half_2 = np.flip(half_2, axis=0)  # 翻轉

    # # resize
    # ratio = half_1.shape[0] / half_2.shape[0]
    # h2, w2 = half_1.shape[0], int(half_2.shape[1]*ratio)
    # half_2 = cv2.resize(half_2, (w2, h2), interpolation=cv2.INTER_CUBIC)

    half_2 = cv2.resize(half_2, (half_2.shape[1], half_1.shape[0]))

    half_1 = fill_color(half_1, 255)
    half_2 = fill_color(half_2, 255)

    return process3(half_1, half_2)

def process3(half_1, half_2):

    half_1 = np.where(half_1 < 120, 0, 255).astype("uint8")
    half_2 = np.where(half_2 < 120, 0, 255).astype("uint8")

    w1, w2 = half_1.shape[1], half_2.shape[1]
    x1, x2 = w1-1, 0

    # 這個部分已經驗證過了，如果用一次多行的方式計算白色比例，最終會有微妙的誤差，
    # 這個微妙的誤差會導致最後decode失敗(真的是龜公)
    for x in range(w1-1, 0, -1):

        if (half_1[:, x] == 255).sum() / (half_1.shape[0]) < 0.7:
            x1 = x
            break
    for x in range(0, w2):
        if (half_2[:, x] == 255).sum() / (half_2.shape[0]) < 0.7:
            x2 = x
            break

    flag = 1
    for x in range(x2, w2//2):
        col = half_2[:, x]
        cnt = 0
        for c in range(1, len(col)):
            if col[c-1] < 120 and col[c] > 120:
                cnt += 1
        if cnt < 9 and flag == 0:
            x2 = x
            break
        if flag == 1 and cnt > 9:
            flag -= 1
        # print(cnt)

    # cv2.namedWindow("half_1", cv2.WINDOW_NORMAL)
    # cv2.imshow('half_1', half_1[:, :x1])
    # cv2.namedWindow("half_2", cv2.WINDOW_NORMAL)
    # cv2.imshow('half_2', half_2[:, x2:])

    qr_code = np.hstack((half_1[:, :x1], half_2[:, x2:]))

    return show(qr_code)

def show(qr_code):
    borders = round(qr_code.shape[0]/21*4)
    canvas = np.zeros(
        (qr_code.shape[0]+2*borders, qr_code.shape[1]+2*borders), dtype=np.uint8)
    canvas[:, :] = 255
    canvas[borders:qr_code.shape[0]+borders,
           borders:qr_code.shape[1]+borders] = qr_code
    # cv2.imshow('qr_code', canvas)
    # cv2.imwrite('qr_code-2.png', canvas)
    # cv2.waitKey(0)

    text = pyzbar.decode(canvas)
    if text == []:
        blur = cv2.GaussianBlur(canvas, (3, 3), 0)
        # closing = cv2.morphologyEx(canvas, cv2.MORPH_CLOSE, np.zeros((3, 3), np.uint8))
        text = pyzbar.decode(blur)
        cv2.imshow('qr_code', blur)

    if text == []:
        return None
    else:
        return text[0].data.decode('utf-8')

def getcorner(canny):
    global width, height
    # shi-tomasi corner detection
    corners = cv2.goodFeaturesToTrack(
        canny, 4, 0.01, width//8, blockSize=10, useHarrisDetector=False, k=0.04)

    # # 另一種方法：亞像素角偵測
    # criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 40, 0.001)
    # cv2.cornerSubPix(canny, np.float32(corners), (5, 5), (-1, -1), criteria)

    corners = np.int0(corners)
    pts = []
    for c in corners:
        x, y = c.ravel()
        pts.append([x, y])

    if len(pts) != 4:
        return None

    def sort(pt):
        pt = sorted(pt, key=lambda x: x[1])
        if pt[0][0] > pt[1][0]:
            pt[0], pt[1] = pt[1], pt[0]
        if pt[3][0] > pt[2][0]:
            pt[2], pt[3] = pt[3], pt[2]
        return pt

    pts = sort(pts)
    # # 畫出角點
    # for x, y in pts:
    #     cv2.circle(canny, (x, y), 3, (0, 0, 255), -1)
    # cv2.imshow('corners', canny)

    return pts

def draw_approx_hull_polygon(img, contours):  # 畫出多邊形
    # 取出多邊形凸包
    hulls = [cv2.convexHull(cnt) for cnt in contours]  # 找輪廓的凸包

    # show image
    img_cp = np.zeros(img.shape, dtype=np.uint8)
    cv2.polylines(img_cp, hulls, isClosed=True, color=255, thickness=1)
    # cv2.imshow('convex hull', img_cp)

    return img_cp

def fill_color(img, color=255):
    h, w = img.shape[:2]
    img = np.asarray(img)
    _, img = cv2.threshold(img, 120, 255, cv2.THRESH_BINARY)
    seedPoint = [(0, 0), (0, h-1), (w-1, h-1), (w-1, 0)]
    for sp in seedPoint:
        cv2.floodFill(img, None, sp, (color, color, color))
    # floodFill(image, mask, seedPoint, newVal, loDiff=None, upDiff=None, flags=None)
    return img

def Perspective(image, pts1, w, h):
    w, h = int(w), int(h)
    if w > h:
        w, h = h, w
    # src: 源點
    pts1 = np.float32(pts1)
    # dst: 目標點（必須以與src點相同的順序列出！）
    pts2 = np.float32([[0, 0], [w, 0], [w, h], [0, h]])

    # 轉換矩陣
    M = cv2.getPerspectiveTransform(pts1, pts2)
    # 套用矩陣
    dst = cv2.warpPerspective(image, M, (w, h))
    return dst


"""
Contains class Density for the calculation of densities of gray images along
segments.

# Author: Vladan Lucic (MPI for Biochemistry)
# $Id$
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from builtins import zip
from builtins import range
#from past.utils import old_div
from past.builtins import basestring

__version__ = "$Revision$"

import sys
import logging
import inspect
from copy import copy, deepcopy

import numpy
import scipy
import scipy.ndimage as ndimage

from .features import Features
from pyto.core.image import Image
from .segment import Segment
from .statistics import Statistics
from .morphology import Morphology

class Density(Features):
    """
    Calculates basic statistics (mean, std, min, max) and volume for
    segments of a (typically greyscale) image.

    Attributes holding calculated data:
      - mean
      - std
      - min
      - max
      - volume

    Each of these data attributes is an ndarray indexed by a segment id (from 1
    up). Elements at position 0 contain values for all segments taken together.

    Methods:
      - calculate(): calculated all data arrays
      - aggregate(): combines calculated data for more than one segment

    Usage:
      density = Density()
      density.calculate(image=grey_image, segments_segmented_image)
    """

    #############################################################
    #
    # Initialization
    #
    #############################################################

    def __init__(self):
        """
        Initializes segments and ids.

        Note (r527): don't want to accept segments as argument because super
        sets the argument segments the the actual array (and loses other info
        like inset).
        """

        # call super
        super(Density, self).__init__()

        # local attributes
        self.dataNames = ['mean', 'std', 'min', 'max', 'volume']

    #############################################################
    #
    # Calculations
    #
    ############################################################

    def calculate(self, image, segments, ids=None):
        """
        Calculates gray value mean, std, min, max and volume for each segment
        from arg segments specified in arg ids. The calculated values are saved
        as attributes named mean, std, min, max and volume, respectivly.

        If arg segments is not specified self.segments is used. If arg ids is
        not specified self.ids is used if found, or if not segments.ids is used.

        Uses minimal possible size of segments and image arrays to speed up
        the calculations, but these arguments are not changed. This method
        respects positioning of image and segments (attribute inset) but
        image.data has to contain the whole inset covered by segments.

        Arguments:
          - image: (core.Image) grayscale image
          - segments: (Segment) segmented image
          - ids: segment ids

        Sets attributes:
          - mean, std (N-1 degrees of fereedom), min, max and volume to
          calculated values
          - ids: to arg ids or segments.ids
        """

        self.setIds(ids=ids, segments=segments)

        # work on array insets
        if self.maxId > 0:
            segments_data, segments_inset = segments.getDataInset()
            segments.makeInset(ids=self.ids)
            image_data, image_inset = image.getDataInset()
            image.useInset(inset=segments.inset, mode='abs')

        # calculate density statistics for each segment
        stats = Statistics(data=image.data, labels=segments.data, ids=self.ids)
        stats.calculate()

        # calculate volume
        mor = Morphology(segments=segments.data, ids=self.ids)
        mor.getVolume()

        # set attributes
        self.mean = stats.mean
        self.std = stats.std
        self.min = stats.min
        self.max = stats.max
        self.volume = mor.volume

        # revert to the original data
        if self.maxId > 0:
            segments.setDataInset(data=segments_data, inset=segments_inset)
            image.setDataInset(data=image_data, inset=image_inset)

    def aggregate(self, ids):
        """
        Returns a new instance with data corresponding to segments combined
        according to arg ids.

        Elements of ids defines segments in the new instance, element i of
        this list defines segment with id i+1 of the new instance. Each element
        of ids should be a list containing segment ids of the current instance
        that are to be taken together.

        For example:

          aggregate(ids=[[1,2], [3,6,7]])

        returns instance with ids 1 and 2, where the new segment 1 contains data
        for actual segments 1 and 3 together, and the new segment 2 for actual
        3, 6 and 7.

        Note that segment ids can be repeated in the argument ids and that not
        all ids need to be present in the argument ids. In those cases
        0-elements of data vectors might be meaningless.

        Argument:
          - ids: (list of lists) segment ids

        Returns: new instance of this class.
        """

        # make new instance
        new = self.__class__()
        new_len = len(ids) + 1
        new.setIds(ids=list(range(1, new_len)))

        # initrialize new data arrays
        new.mean = numpy.zeros(new_len) - 1
        new.std = numpy.zeros(new_len) - 1
        new.min = numpy.zeros(new_len) - 1
        new.max = numpy.zeros(new_len) - 1
        new.volume = numpy.zeros(new_len, dtype=int) - 1
        sum_sq = numpy.zeros(new_len) - 1

        # agregated segments
        for old_ids, new_id in zip(ids, new.ids):
            new.volume[new_id] = self.volume[old_ids].sum()
            new.mean[new_id] = (
                numpy.inner(self.mean[old_ids], self.volume[old_ids])
                / new.volume[new_id].astype(float))
            new.min[new_id] = self.min[old_ids].min()
            new.max[new_id] = self.max[old_ids].max()

            # std
            ddof = 0.
            sum_sq[new_id] = \
                ((self.std[old_ids] ** 2 * (self.volume[old_ids] - ddof))
                 + self.mean[old_ids] ** 2 * self.volume[old_ids]).sum()
            new.std[new_id] = numpy.sqrt(
                (sum_sq[new_id] - new.mean[new_id] ** 2 * new.volume[new_id])
                / (new.volume[new_id] - ddof))

        # set total (index 0) values
        new.setTotal()

        return new

    def restrict(self, ids):
        """
        Effectivly removes data that correspond to ids that are not specified
        in the arg ids. Arg ids should not contain ids that are not in self.ids.

        Sets self.ids to ids and recalculates totals (index 0). Currently
        it doesn't actually remove non-id values from data.

        Argument:
          - ids: new ids
        """

        # set ids
        super(Density, self).restrict(ids=ids)

        # remove non-id values?

        # set total
        self.setTotal()

    def calculateTotal(self):
        """
        Calculates total values based on the individual segments data
        and saves them at index 0.

        Acts on attributes min, max, volume, mean and std, if in self.dataNames.

        Doesn't do anything if there are no ids (self.ids).
        """

        # don't do anything if no ids
        if len(self.ids) == 0: return

        if 'min' in self.dataNames:
            self.min[0] = self.min[self.ids].min()
        if 'max' in self.dataNames:
            self.max[0] = self.max[self.ids].max()
        if 'volume' in self.dataNames:
            self.volume[0] = self.volume[self.ids].sum()

        # mean
        if ('mean' in self.dataNames) and ('volume' in self.dataNames):
            self.mean[0] = (
                numpy.dot(self.mean[self.ids], self.volume[self.ids])
                / float(self.volume[0]))

        # std
        if (('std' in self.dataNames) and ('mean' in self.dataNames)
            and ('volume' in self.dataNames)):
            variances = numpy.dot(
                (self.std[self.ids]**2 + self.mean[self.ids]**2),
                self.volume[self.ids])
            means = numpy.dot(
                self.mean[self.ids], self.volume[self.ids])
            self.std[0] = numpy.sqrt(
                variances / float(self.volume[0])
                - (means / float(self.volume[0]))**2)

    def merge(
            self, new, names=None, mode='replace', mode0='consistent',
            volume=None):
        """
        Merges data of this object new with the data of this instance.
        The values of all attributes listed in self.dataNames are added.

        The data attributes whose values are meged are those listed in names or
        in self.dataNames if names is None.

        If mode is 'add' the data is simply added. If mode is
        'replace' the new values replace the old ones for id that are in
        new.ids.

        Recommended value for arg mode0 is 'consistent', because in this case
        index 0 of arrays mean, std min and max will contain appropriate values
        for all segments taken together.

        Arguments:
          - new: instance of Morphology
          - names: (list of strings, or a single string) names of data
          attributes
          - mode: merge mode for data (indices 1+) 'add' or 'replace'
          - mode0: merge mode for index 0, 'consistent', 'add', 'replace'
          or a numerical value
        """

        #
        mode0_consistent = False
        if isinstance(mode0, basestring) and (mode0 == 'consistent'):
            mode0_consistent = True
            mode0 = -1

        # merge data listed in self.dataNames
        super(Density, self).merge(
            new=new, names=names, mode=mode, mode0=mode0)

        # set volume
        if (volume is None) and ('volume' in self.dataNames):
            volume = self.volume

        # calculate index 0 values if needed
        if mode0_consistent:
            #if 'min' in self.dataNames:
            #    self.min[0] = self.min[self.ids].min()
            #if 'max' in self.dataNames:
            #    self.max[0] = self.max[self.ids].max()
            #if 'volume' in self.dataNames:
            #    self.volume[0] = self.volume[self.ids].sum()
            #if ('mean' in self.dataNames) and (volume is not None):
            #    total_dens = numpy.dot(self.mean[self.ids], volume[self.ids])
            #    self.mean[0] = total_dens / float(self.volume[0])
            #if ('std' in self.dataNames) and (volume is not None):
            #    variances = numpy.dot(
            #        (self.std[self.ids]**2 + self.mean[self.ids]**2),
            #        self.volume[self.ids])
            #    means = numpy.dot(
            #        self.mean[self.ids], self.volume[self.ids])
            #    self.std[0] = numpy.sqrt(
            #        variances / float(self.volume[0])
            #        - (means / float(self.volume[0]))**2)
            self.calculateTotal()

    def setTotal(self):
        """
        Sets total values (index 0) for all data.

        In particular, total volume is the sum of all volumes, while total mean,
        std, min and max are values obtained for all segments taken together
        (and not mean of the segment means and so on. Takes into account only
        segments listed in self.ids.

        Sets index 0 of data arrays.
        """

        ids = self.ids

        # don't do anything if no ids
        if (ids is None) or (len(ids) == 0):
            return

        # volume
        self.volume[0] = self.volume[ids].sum()

        # mean, min, max
        self.mean[0] = (
            numpy.inner(self.mean[ids], self.volume[ids])
            / float(self.volume[0]))
        self.max[0] = self.max[ids].max()
        self.min[0] = self.min[ids].min()

        # std
        ddof = 0
        sum_sq = (self.volume[ids] - ddof) * self.std[ids]**2 \
            + self.volume[ids] * (self.mean[ids] - self.mean[0])**2
        self.std[0] = numpy.sqrt(sum_sq.sum() / (self.volume[0] - ddof))


import torch
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt
import torch.nn.functional as F
import time
from torch.optim.lr_scheduler import ReduceLROnPlateau

## interactive off
plt.ioff()
## setup torch enviro
torch.manual_seed(42)
torch.cuda.manual_seed_all(42)
torch.autograd.set_detect_anomaly(True)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def single_to_multi_label(y):
  """
    Input: labels in {0,...,9}
    Output: one-hot encoding of y
  """
  y_onehot = torch.FloatTensor(y.shape[0], 10)
  y = y.unsqueeze(1)
  y_onehot.zero_()
  y_onehot.scatter_(1, y, 1)
  return y_onehot

## PLOT HELPER
def plot_history(key, train, history):
  """
    Plot loss and accuracy history during model run
    Input:
          key : str => name of the model
          train : bool => training 1 or test 0
          history : dict{str : list of floats}
  """
  if train:
    when = "train"
  else:
    when = "test"

  fig, ax = plt.subplots( 1, 2, figsize = (12,4) )
  ax[0].plot(history['loss'], label = "loss")
  ax[0].set_title( "Loss" )
  ax[0].set_xlabel( "Epochs" )
  ax[0].set_ylabel( "Loss" )
  ax[0].grid( True )
  ax[0].legend()

  ax[1].plot(history['accuracy'], label = "accuracy")
  ax[1].set_title( "Accuracy" )
  ax[1].set_xlabel( "Epochs" )
  ax[1].set_ylabel( "Accuracy" )
  ax[1].grid( True )
  ax[1].legend()

  plt.savefig(f"./images/{key}_{when}.png")
  plt.close()

## TRAIN
def train(logger, model, dataHandler, num_epochs, lr=0.001, momentum=0.9, l1_ratio=0.5, l1l2_penalty=0.0000001, l1_penalty=0.01, l2_penalty=0.01, optim_algo='SGD', loss='MSE', regularizer='None'):
  print("Using: ", device)
  if regularizer == 'None':
    l2_penalty = 0
    l1_penalty = 0
  elif regularizer == 'L1':
    l2_penalty = 0
  elif regularizer == 'L2':
    l1_penalty = 0
  elif regularizer == 'Elastic':
    l1_penalty = l1_ratio*l1l2_penalty
    l2_penalty = l1l2_penalty*(1-l1_ratio)
  else:
    print("regularizer has to be None, L1, L2 or Elastic")
    exit()

  if optim_algo == 'Adam':
    optimizer = optim.Adam(model.parameters(),lr=lr, weight_decay=l2_penalty, amsgrad=True)
  elif optim_algo == 'SGD':
    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=l2_penalty, nesterov=True)
  else:
    print("Optimizer is Adam or SGD")
    exit
  if loss == 'CSE':
    criterion = nn.CrossEntropyLoss()
  elif loss == 'MSE':
    criterion = nn.MSELoss()
  else:
    print("Loss is CrossEntropy or MSE")
    exit

  scheduler = ReduceLROnPlateau(optimizer, 'min'
                                  ,patience=3,factor=0.9817
                                 ,verbose=True,threshold=1e-2)

  print(f"Training summary:\n lr={lr}\n epochs={num_epochs}\n optim={optim_algo}\n loss={loss}\n regularizer={regularizer}")

  trainHistory = {}
  trainHistory['loss'] = []
  trainHistory['accuracy'] = []

  model.to(device)
  model.train()
  for epoch in range(num_epochs):

    start = time.time()
    epoch_loss = 0
    num_correct = 0.0
    num_samples = 0.0

    for i, (data, labels) in enumerate(dataHandler.train_dl):
      data = data.to(device=device)
      if loss == 'MSE':
        labels_one_hot = single_to_multi_label(labels)
        labels_one_hot = labels_one_hot.to(device=device)
      labels = labels.to(device)

      optimizer.zero_grad()

      predictions = model(data)
      if loss == 'MSE':
        #predictions, labels_one_hot = predictions.type('torch.FloatTensor'), labels_one_hot.type('torch.FloatTensor')
        loss_value = criterion(predictions, labels_one_hot)
      else:
        #predictions, labels = predictions.type('torch.FloatTensor'), labels.type('torch.FloatTensor')
        loss_value = criterion(predictions, labels)

      if regularizer == 'L1':
        parameters = []
        for parameter in model.parameters():
          parameters.append(parameter.view(-1))
        l1_norm = torch.abs(torch.cat(parameters)).sum()
        loss_value = loss_value + l1_penalty*l1_norm

      loss_value.backward()
      optimizer.step()

      epoch_loss += loss_value.item()
      _, predicted_labels = predictions.max(1)
      correct = (predicted_labels == labels).sum().item()
      samples = predicted_labels.size(0)
      num_correct += correct
      num_samples += samples

      if model.verbose and (i+1)%100 == 0:
        print(f"[?] Step {i+1}/{len(dataHandler.train_dl)} Epoch {epoch+1}/{num_epochs} Loss {loss_value.item()} Accuracy {correct/samples:.4f}")

    end = time.time()
    print(f"[?] {logger.name} Epoch {epoch+1}/{num_epochs} Loss {epoch_loss/(i+1):.4f} Accuracy {num_correct/num_samples:.4f} Time: {end-start:.4f}s")
    logger.log_step(epoch, i, epoch_loss/(i+1), num_correct/num_samples)
    trainHistory['loss'].append(epoch_loss/(i+1))
    trainHistory['accuracy'].append(num_correct/num_samples)

    scheduler.step(epoch_loss/(i+1))

  plot_history(logger.name, True, trainHistory)

## EVAL
def eval(logger, model, dataHandler, loss='MSE'):
  num_correct = 0
  num_samples = 0

  if loss == 'CSE':
    criterion = nn.CrossEntropyLoss()
  elif loss == 'MSE':
    criterion = nn.MSELoss()
  else:
    print("Loss is CrossEntropy or MSE")
    exit

  model.eval()

  testHistory = {}
  testHistory['loss'] = []
  testHistory['accuracy'] = []
  test_loss = 0
  test_accuracy = 0

  with torch.no_grad():
    for batch, (data,labels) in enumerate(dataHandler.test_dl):
        data = data.to(device=device)
        if loss == 'MSE':
          labels_one_hot = single_to_multi_label(labels)
          labels_one_hot = labels_one_hot.to(device=device)
        else:
          labels = labels.to(device=device)

        predictions = model(data)
        if loss == 'MSE':
          predictions, labels_one_hot = predictions.type('torch.FloatTensor'), labels_one_hot.type('torch.FloatTensor')
          loss_value = criterion(predictions, labels_one_hot).item()
        else:
          predictions = model(data)
          #predictions, labels = predictions.type('torch.FloatTensor'), labels.type('torch.FloatTensor')
          loss_value = criterion(predictions, labels).item()

        test_loss += loss_value
        _, predicted_labels = predictions.max(1)
        num_correct += (predicted_labels == labels).sum().item()
        num_samples += predicted_labels.size(0)

        testHistory['loss'].append(loss_value)
        testHistory['accuracy'].append(float(num_correct) / float(num_samples))

        logger.log_batch(batch+1, loss_value, float(num_correct) / float(num_samples))

    test_accuracy = float(num_correct) / float(num_samples)
    test_loss = test_loss/len(dataHandler.test_dl) ## avg loss
    logger.finalize(test_loss, test_accuracy)

    plot_history(logger.name, False, testHistory)

  return test_loss, test_accuracy


from typing import Any

import fastapi

from schemas.documentation import DocumentationOut

router = fastapi.APIRouter()

@router.get("/documentation", response_model=DocumentationOut)
def get_doc() -> dict[str, Any]:
    docs: dict[str, Any] = {
        "model_name": "image_classification_vissl",
        "short_desc": (
            "ViSSL is a computer vision library for state-of-the-art deep learning"
            " models. It provides pre-trained models, tools for training new models,"
            " and a framework for building custom models."
        ),
        "ideal": (
            "Ideal for computer vision tasks that require high accuracy and advanced"
            " features, such as object detection, semantic segmentation, and instance"
            " segmentation."
        ),
        "not_ideal": (
            "May not be ideal for low-resource or real-time applications due to the"
            " computational intensity of the models."
        ),
        "input_args": {"example1": "Image file in a format supported by the library."},
        "output_args": {"example1": "BMP"},
    }
    return docs


import subprocess
from contextlib import asynccontextmanager

from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.exc import NoSuchTableError

from contextqa import settings

def _migrations_already_applied() -> bool:
    engine = create_engine(settings.sqlalchemy_url)
    metadata = MetaData()

    # Pass the 'bind' argument when creating the Table
    try:
        Table("alembic_version", metadata, autoload_with=engine)
    except NoSuchTableError:
        return False
    return True

@asynccontextmanager
async def check_migrations(*_):
    """Apply migrations if necessary"""
    if not _migrations_already_applied():
        code = subprocess.check_call(["alembic", "upgrade", "head"])
        print("Migration finished with exit code:", code)
    yield
    print("Terminating...")