import os, sys
import logging
import multiprocessing as mp

from argparse import ArgumentParser
from collections import OrderedDict

sys.path.insert(0, os.path.dirname(os.getcwd()))
from Core.CoreSystem import  Helper

class clsParameters():

    def __init__(self, options):
        self.strUser    = options.user_name
        self.strProject = options.project_name.replace('.txt', '') ## A user can be confused the input. So I prevented from it using 'replace'.
        self.strGroup   = options.group
        self.intCore    = options.thread

        self.strSampleList = 'User/{user}/{project}.txt'.format(user=options.user_name, project=options.project_name)

def SummaryRandomBarcode(sFile_path):

    """
    /Tmp
    190819_Nahye_24k_2_D0_2-24kLib_Classified_Indel_barcode.fastq* -> process target
    190819_Nahye_24k_2_D0_2-24kLib_Indel_freq.txt*
    190819_Nahye_24k_2_D0_2-24kLib_Indel_summary.txt*
    190819_Nahye_24k_2_D0_2-24kLib_Summary.txt*
    Pickle

    dBarcode_cnt = {'ACGTACTC_sorting_barcode': {'ACATACAC_random': 5, 'CGTGTTGA_random': 3, ...}
    """
    dictBarcodeCnt = {}
    strClassCheck  = ''

    strSample = sFile_path.split('/')[-1]
    logging.info('Summary_random_barcode start : %s, %s' % (sFile_path, strSample))

    for sFile in os.listdir(sFile_path+'/Tmp/'):
        if '.fastq' in sFile:
            with open(sFile_path+'/Tmp/'+sFile) as Input:
                for i, strRow in enumerate(Input):

                    # @D00235:683:CE1P6ANXX:6:1114:2135:5231 1:N:0:CTGAAGCT+CCTATCCT:Barcode_TTTGCTATCTCGACGTATGGACAGTG:total
                    if i % 4 == 0:
                        listBarClass = strRow.replace('\n','').split('Barcode_')[1].split(':')
                        strBarcode   = listBarClass[0]
                        strClass     = listBarClass[1]

                        if strClass == 'total':
                            strClassCheck = 'total'

                    if i % 4 == 1 and strClassCheck == 'total':
                        strRow = strRow.replace('\n','').upper()
                        intBarcodeStart   = strRow.find(strBarcode)
                        strRandom_barcode = strRow[intBarcodeStart-8:intBarcodeStart]

                        try:
                            _ = dictBarcodeCnt[strBarcode]
                        except KeyError:
                            dictBarcodeCnt[strBarcode] = {}
                        try:
                            dictBarcodeCnt[strBarcode][strRandom_barcode] += 1
                        except KeyError:
                            dictBarcodeCnt[strBarcode][strRandom_barcode] = 1
                        #print(sBarcode, sRandom_barcode, iBarcode_start, sRow)

                        strClassCheck = ''

    if not os.path.isdir(sFile_path + '/Summary_Random_barcode'): os.mkdir(sFile_path + '/Summary_Random_barcode')
    with open(sFile_path + '/Summary_Random_barcode/%s_all_random_barcode.txt' % strSample, 'w') as All_random,\
        open(sFile_path + '/Summary_Random_barcode/%s_Unique_RandomBarcodeNumber_In_SortingBarcode.txt' % strSample, 'w') as Random_sorting:

        All_random.write('Sorting_barcode\tUnique_RandomBarcodeNumber_In_SortingBarcode\tRandomBarcode\tEach_RandomBarcode_read_count\n')
        Random_sorting.write('Sorting_barcode\tUnique_RandomBarcodeNumber_In_SortingBarcode\n')

        for sBarcode, dRandom_barcode_cnt in dictBarcodeCnt.items():
            iRandom_barcode_num = len(dRandom_barcode_cnt.keys())
            Random_sorting.write('\t'.join(map(str, [sBarcode, iRandom_barcode_num]))+'\n')

            for sRandom_barcode, iCnt in dRandom_barcode_cnt.items():
                All_random.write('\t'.join(map(str, [sBarcode, iRandom_barcode_num, sRandom_barcode, iCnt]))+'\n')

    logging.info('Summary_random_barcode end: %s' % sFile_path)

## on going
def CountGroup(InstParameters):
    """
    Sorting_barcode Unique_RandomBarcodeNumber_In_SortingBarcode    RandomBarcode   Each_RandomBarcode_read_count
    TATATCATAGCGTACTCATC    8       TGCGTTTG        3
    TATATCATAGCGTACTCATC    8       CGCGTTTG        3
    TATATCATAGCGTACTCATC    8       TAGTTTTG        1
    TATATCATAGCGTACTCATC    8       ATAGTTTG        1
    """

    sHeader = ''

    with open(InstParameters.strSampleList) as Sample: ## tmp input

        listSample = Sample.readlines()

        setGroup = set([strRow.replace('\n', '').split('\t')[2].upper() for strRow in listSample])

        for strGroup in setGroup:
            if strGroup == 'CTRL': continue

            for strRow in listSample:
                if strGroup == strGroupOfSample:  ## matched group names -> Sum the counts
                    listCol          = strRow.replace('\n', '').split('\t')
                    strSample        = listCol[0]
                    strRef           = listCol[1]
                    strGroupOfSample = listCol[2]

                    strProjectDir = './Output/{user}/{project}'.format(user=InstParameters.strUser,
                                                                       project=InstParameters.strProject)
                    strGroupDir = os.path.join(strProjectDir, 'Group_result')
                    Helper.MakeFolderIfNot(strGroupDir)

                    dTotal_RandomBarcode_cnt_in_SortingBarcode = OrderedDict() ## ('GECKO_6367_GATCTGCTC', ['GECKO_6367', 'GATCTGCTC', 2, 156, '0.0128']),
                                                                               ## Unique key, only one list.

                    with open('{project_dir}/{sample}_all_random_barcode.txt'.format(project_dir=strProjectDir,
                                                                                sample=strSample)) as RandomBarcode_SeqFreq:
                        sHeader = RandomBarcode_SeqFreq.readline()

                        for sRow in RandomBarcode_SeqFreq:
                            lCol = sRow.replace('\n', '').split('\t')

                            sSortingBarcode                             = lCol[0]
                            #iTotal_RandomBarcode_cnt_in_SortingBarcode  = int(lCol[1])
                            sSorting_and_Random_barcode_seq             = lCol[0] + '_' + lCol[2]  ## Unique name : Doench2014_1000_CTCTGGGGT
                            iRandomBarcode_count                        = int(lCol[3])

                            lCol[3] = iRandomBarcode_count

                            try:
                                _ = dTotal_RandomBarcode_cnt_in_SortingBarcode[sSorting_and_Random_barcode_seq]

                                dTotal_RandomBarcode_cnt_in_SortingBarcode[sSorting_and_Random_barcode_seq][3] += iRandomBarcode_count

                            except KeyError:
                                dTotal_RandomBarcode_cnt_in_SortingBarcode[sSorting_and_Random_barcode_seq] = lCol  ## initial assignment
                    #END for
                    dRecal_total_kind_of_RandomBarcode = OrderedDict()
                    for sSort_Rand_seq in dTotal_RandomBarcode_cnt_in_SortingBarcode:  ## sSorting_and_Random_barcode_seq
                        sSortBarcode = sSort_Rand_seq.split('_')[0]
                        try:
                            dRecal_total_kind_of_RandomBarcode[sSortBarcode].append(dTotal_RandomBarcode_cnt_in_SortingBarcode[sSort_Rand_seq])
                        except KeyError:
                            dRecal_total_kind_of_RandomBarcode[sSortBarcode] = [dTotal_RandomBarcode_cnt_in_SortingBarcode[sSort_Rand_seq]]

                    for sKey, llValue in dRecal_total_kind_of_RandomBarcode.items():
                        ## sKey: TATATCATAGCGTACTCATC, llValue : [[TATATCATAGCGTACTCATC, 8, TGCGTTTG, 3],[],[] ...
                        iKind_of_RandomBarcode = len(llValue)  ################## why do I make like this ?????
                        for lValue in llValue:
                            lValue[1] = iKind_of_RandomBarcode ## Recal using group total cnt.

                        llValue = sorted(llValue, key=lambda x:x[3], reverse=True)
                        dRecal_total_kind_of_RandomBarcode[sKey] = llValue

                    strEachGroup = './Output/Group_result/%s' % strGroup
                    Helper.MakeFolderIfNot(strEachGroup)

                    with open(os.path.join(strEachGroup, 'Summary_all_random_barcode_in_group.txt'), 'w') as Sort_Random_cnt,\
                        open(os.path.join(strEachGroup, 'Summary_Unique_RandomBarcodeNumber_in_group.txt'), 'w') as Uniq_random_cnt:

                        Sort_Random_cnt.write(sHeader)
                        Uniq_random_cnt.write('Sorting_barcode\tUnique_RandomBarcodeNumber_In_SortingBarcode\n')

                        for sSortBarcode, llCol in dRecal_total_kind_of_RandomBarcode.items():
                            Uniq_random_cnt.write('\t'.join(map(str, [sSortBarcode, len(llCol)]))+'\n')
                            for lCol in llCol:
                                Sort_Random_cnt.write('\t'.join(map(str, lCol))+'\n')
        #END: for
    #END: with

def Main():

    logging.info('Program Start')
    logging.info('Make commands for a multiple processing')

    parser = ArgumentParser(description='Script for counting the random barcodes')

    parser.add_argument('-u', '--user_name', type=str, dest='user_name', help='The user name in the /user subdir')
    parser.add_argument('-p', '--project_name', type=str, dest='project_name', help='The project name in the /user/user_name/ subdir')
    parser.add_argument('-g', '--group',  type=str, dest='group', default='false', help='The group sum run of the barcodes, default: false')
    parser.add_argument('-t', '--thread', type=int, dest='thread', default='15', help='The multicore number 1~15')
    options = parser.parse_args()

    InstParameters = clsParameters(options)

    lPara = []

    with open(InstParameters.strSampleList) as SampleList:

        for strSample in SampleList:
            if strSample[0] == '#' or strSample[0] in ['', ' ', '\r', '\n', '\r\n']: continue
            strSample  = strSample.replace('\n', '').replace('\r', '').split('\t')[0]
            sFile_path = './Output/{user}/{project}/{sample}'.format(user=options.user_name,
                                                                     project=options.project_name,
                                                                     sample=strSample)
            #print('sFile_path', sFile_path)
            lPara.append(sFile_path)

    ## single_test
    #Summary_random_barcode(lPara[0])

    logging.info('Multiple processing Start')
    p = mp.Pool(options.thread)
    p.map_async(SummaryRandomBarcode, lPara).get()
    logging.info('Multiple processing End')

    #logging.info('Count group Start')
    #CountGroup(InstParameters)
    #logging.info('Count group End')

    #logging.info('Program End')

Main()


import logging
logger = logging.getLogger(__name__)

import os
import re
import yaml
import json
from comment_parser import comment_parser

class Parser():
    def __init__(self, threatmodel):

        self.threatmodel = threatmodel

        self.action_table = {}
        self.action_table["mitigate"] = self.threatmodel.add_mitigation
        self.action_table["accept"] = self.threatmodel.add_acceptance
        self.action_table["transfer"] = self.threatmodel.add_transfer
        self.action_table["expose"] = self.threatmodel.add_exposure
        self.action_table["connect"] = self.threatmodel.add_connection
        self.action_table["review"] = self.threatmodel.add_review
        self.action_table["test"] = self.threatmodel.add_test
        self.action_table["threat"] = self.threatmodel.add_threat
        self.action_table["control"] = self.threatmodel.add_control
        self.action_table["component"] = self.threatmodel.add_component

        self.patterns = {}
        self.patterns["mitigate"] = r'@mitigates? (?P<component>.*?) against (?P<threat>.*?) with (?P<control>.*)'
        self.patterns["accept"] = r'@accepts? (?P<threat>.*?) to (?P<component>.*?) with (?P<details>.*)'
        self.patterns["transfer"] = r'@transfers? (?P<threat>.*?) from (?P<source_component>.*?) to (?P<destination_component>.*?) with (?P<details>.*)'
        self.patterns["expose"] = r'@exposes? (?P<component>.*?) to (?P<threat>.*?) with (?P<details>.*)'
        self.patterns["connect"] = r'@connects? (?P<source_component>.*?) (?P<direction>with|to) (?P<destination_component>.*?) with (?P<details>.*)'
        self.patterns["review"] = r'@reviews? (?P<component>.*?) (?P<details>.*)'
        self.patterns["test"] = r'@tests? (?P<control>.*?) for (?P<component>.*)'

        self.patterns["threat"] = r'@threat (?P<threat>.*)'
        self.patterns["control"] = r'@control (?P<control>.*)'
        self.patterns["component"] = r'@component (?P<component>.*)'

        self.cwd = os.getcwd()

    def run_action(self, data, source):
        action = data.pop("action")
        self.action_table[action](data, source=source)

    def is_extended(self, line):
        return line[-1] == ":"

    def is_threatspec_line(self, line):
        for key in self.patterns.keys():
            if "@{}".format(key) in line:
                return True
        return False

    def check_file(self, filename):
        logger.debug("Parsing file {}".format(filename))
        if filename.startswith(self.cwd):
            return filename.replace(self.cwd, "", 1).lstrip(os.path.sep)
        return filename

class CommentParser(Parser):
    def __init__(self, threatmodel, mime=None):
        super().__init__(threatmodel)
        self.mime = mime

    def parse_comment(self, comment):
        annotations = []

        LINE = 0
        EXTENDED = 1

        state = LINE
        extended_lines = []
        data = None

        line_number = 1

        for line in comment.split("\n"):
            stripped_line = self.strip(line)
            if state == LINE:
                for action in self.patterns.keys():
                    if stripped_line.startswith("@" + action):
                        data = {"action": action, "line": line_number, "annotation": stripped_line}
                        extended_lines = []
                        pattern = self.patterns[action]
                        if self.is_extended(stripped_line):
                            state = EXTENDED
                            stripped_line = stripped_line[0:-1]
                        m = re.match(pattern, stripped_line, re.M | re.I)
                        if m:
                            data.update(m.groupdict())
                            if state == LINE:
                                annotations.append(data)
                        else:
                            raise Exception("Could not parse {} pattern:\n{} for comment line:\n{}".format(action, pattern, line))

            elif state == EXTENDED:
                if stripped_line == "":
                    state = LINE
                    self.process_extended_lines(extended_lines, data, annotations)
                    extended_lines = []
                else:
                    extended_lines.append(self.strip_stars(line))

            line_number += 1

        if len(extended_lines) > 0:
            self.process_extended_lines(extended_lines, data, annotations)

        return annotations

    def process_extended_lines(self, extended_lines, data, annotations):
        extended_text = "\n".join(extended_lines)
        data["annotation"] += "\n" + extended_text
        data.update(yaml.load(extended_text, Loader=yaml.SafeLoader))
        annotations.append(data)

    def strip(self, line):
        return self.strip_stars(line).strip()

    def strip_stars(self, line):
        if self.mime not in ["text/html", "text/x-shellscript", "text/xml"]:
            return re.sub(r"\s*\*+", "", line)
        return line

class TextFileParser(CommentParser):
    def parse_file(self, filename):
        filename = self.check_file(filename)

        with open(filename) as fh:
            data = fh.read()

        source = {
            "filename": filename,
            "code": ""
        }

        for data in self.parse_comment(data):
            source["annotation"] = data.pop("annotation")
            source["line"] = data.pop("line")
            self.run_action(data, source)

class SourceFileParser(CommentParser):

    def __init__(self, threatmodel, mime=None):
        super().__init__(threatmodel)
        self.mime = mime

    def extract_comment_context(self, lines, commented_lines, start_line, num_lines, multiline=False):
        count = 0
        i = start_line
        code = []

        capture_first_line = not multiline

        for line in lines[start_line - 1:]:
            if count >= num_lines:
                return "".join(code)

            if capture_first_line:
                code.append(line)
                capture_first_line = False

            if i not in commented_lines:
                code.append(line)
                count += 1
            i += 1
        return "".join(code)

    def get_lines(self, filename):
        try:
            with open(filename) as fh:
                return fh.readlines()
        except UnicodeDecodeError:
            return None

    def parse_file(self, filename):
        logger.debug("Parsing file {}".format(filename))

        lines = self.get_lines(filename)
        if not lines:
            return

        commented_line_numbers = []
        comments = []
        try:
            for comment in comment_parser.extract_comments(filename, self.mime):
                comment_text = comment.text()
                comment_line = comment.line_number()
                if comment.is_multiline():
                    offset = len(comment_text.split("\n"))
                    commented_line_numbers += range(comment_line, comment_line + offset)
                else:
                    offset = 0
                    commented_line_numbers.append(comment_line)
                comments.append({
                    "text": comment_text,
                    "line": comment_line,
                    "offset": offset,
                    "multiline": comment.is_multiline()
                })
        except comment_parser.UnsupportedError as e:
            print(e)
            return

        for comment in comments:
            comment["text"] = comment["text"].strip()
            num_lines = 5  # Get 5 lines of code
            code = self.extract_comment_context(lines, commented_line_numbers, comment["line"] + comment["offset"], num_lines, comment["multiline"])

            source = {
                "code": code,
                "filename": filename
            }

            annotations = self.parse_comment(comment["text"])
            if annotations:
                for data in annotations:
                    source["line"] = data.pop("line")
                    source["annotation"] = data.pop("annotation")
                    self.run_action(data, source)

class YamlFileParser(Parser):
    def parse_annotation(self, annotation, data={}):
        stripped_line = annotation.strip()
        for action in self.patterns.keys():
            if stripped_line.startswith("@" + action):
                data["action"] = action
                pattern = self.patterns[action]
                m = re.match(pattern, stripped_line, re.M | re.I)
                if m:
                    data.update(m.groupdict())
                    return data
                else:
                    raise Exception("Could not parse {} pattern:\n{} for comment line:\n{}".format(action, pattern, stripped_line))

    def parse_key(self, data, parent, filename):
        if isinstance(data, str):
            annotation = self.parse_annotation(data)
            source = {
                "annotation": data,
                "code": json.dumps(parent, indent=2),
                "filename": filename,
                "line": 0
            }
            self.run_action(annotation, source)
        elif isinstance(data, list):
            for v in data:
                if not isinstance(v, str):
                    raise Exception("Invalid value type for x-threatspec list in {}".format(filename))
                annotation = self.parse_annotation(v)
                source = {
                    "annotation": v,
                    "code": json.dumps(parent, indent=2),
                    "filename": filename,
                    "line": 0
                }
                self.run_action(annotation, source)
        elif isinstance(data, dict):
            for k, v in data.items():
                annotation = self.parse_annotation(k, v)
                source = {
                    "annotation": k,
                    "code": json.dumps(parent, indent=2),
                    "filename": filename,
                    "line": 0
                }
                self.run_action(annotation, source)

    def parse_data(self, data, parent, filename):
        if isinstance(data, dict):
            for k, v in data.items():
                if k == "x-threatspec":
                    self.parse_key(v, data, filename)
                else:
                    self.parse_data(v, data, filename)
        elif isinstance(data, list):
            for v in data:
                self.parse_data(v, data, filename)

    def parse_file(self, filename):
        filename = self.check_file(filename)

        with open(filename) as fh:
            file_data = yaml.load(fh, Loader=yaml.SafeLoader)
            self.parse_data(file_data, {}, filename)


from adestis_netbox_plugin_account_management.models import *
from netbox.filtersets import NetBoxModelFilterSet
from django.db.models import Q

from dcim.models import *
from utilities.filters import TreeNodeMultipleChoiceFilter
from django.utils.translation import gettext as _
import django_filters
from utilities.forms import (
    DynamicModelMultipleChoiceField, MultipleChoiceField, StaticSelect, TagFilterField, BOOLEAN_WITH_BLANK_CHOICES,
)
from virtualization.models import VirtualMachine, ClusterGroup, Cluster
from tenancy.models import *

__all__ = (
    'SystemFilterSet',
)

class SystemFilterSet(NetBoxModelFilterSet):

    cluster_group_id = DynamicModelMultipleChoiceField(
        queryset=ClusterGroup.objects.all(),
        required=False,
        label=_('Cluster group (name)')
    )

    cluster_id = DynamicModelMultipleChoiceField(
        queryset=Cluster.objects.all(),
        required=False,
        label=_('Cluster (name)')
    )

    device_id = django_filters.ModelMultipleChoiceFilter(
        queryset=Device.objects.all(),
        label=_('Device (ID)'),
    )

    device = django_filters.ModelMultipleChoiceFilter(
        field_name='device__name',
        queryset=Device.objects.all(),
        to_field_name='name',
        label=_('Device (name)'),
    )

    virtual_machine_id = DynamicModelMultipleChoiceField(
        queryset=VirtualMachine.objects.all(),
        required=False,
        label=_('Virtual machine (name)'))

    group = TreeNodeMultipleChoiceFilter(
        queryset=TenantGroup.objects.all(),
        field_name='group',
        lookup_expr='in',
        to_field_name='group',
        label=_('Tenant group (group)'),
    )

    tenant_id = django_filters.ModelMultipleChoiceFilter(
        queryset=Tenant.objects.all(),
        label=_('Tenant (ID)'),
    )

    tenant = django_filters.ModelMultipleChoiceFilter(
        queryset=Tenant.objects.all(),
        field_name='tenant__name',
        to_field_name='tenant',
        label=_('Tenant (name)'),
    )

    class Meta:
        model = System
        fields = ('id', 'tenant', 'group', 'cluster_group_id', 'cluster_id', 'device', 'virtual_machine_id', 'name', 'system_url', 'system_status')

    def search(self, queryset, name, value):
        if not value.strip():
            return queryset
        return queryset.filter(
            Q(name__icontains=value) |
            Q(system_url__icontains=value) |
            Q(system_status__icontains=value)
        )


import numpy as np
import pandas as pd
from tqdm.notebook import trange

def vuln_sim(dists, sim_years, days_in_year=365, sample_count=1000000, clip=True, verbose=False):
    samples = pd.DataFrame({key:val.rvs(sample_count) for key, val in dists.items()})

    variants = {}
    active = []
    vuln_days = []
    t = trange(sim_years * days_in_year) if verbose else range(sim_years * days_in_year)
    for i in t:
        var_occur = np.random.choice(samples['occurence']) / days_in_year
        if var_occur > np.random.uniform():
            var_idx = len(variants)
            variants[var_idx] = {
                'start_day': i,
                'identification': np.random.choice(samples['identification']),
                'remediation': np.random.choice(samples['remediation']),
            }
            variants[var_idx]['duration'] = variants[var_idx]['identification'] + variants[var_idx]['remediation']
            active.append(var_idx)

        efficacy = np.random.choice(samples['variant']) if len(active) > 0 \
            else np.random.choice(samples['efficacy'])

        if efficacy < np.random.uniform():
            vuln_days.append(i)

        for var_idx in [*active]:
            var_end = 1 / variants[var_idx]['duration']
            if var_end > np.random.uniform():
                active.remove(var_idx)
                variants[var_idx]['end_day'] = i

    vuln_vals = (pd.Series(vuln_days) // days_in_year).value_counts().reindex(np.arange(1000)).fillna(0) / days_in_year
    if clip: vuln_vals = vuln_vals.clip(1/sim_years, 1 - (1/sim_years))

    return vuln_vals, variants


import firebase_admin
from firebase_admin import credentials
from firebase_admin import db
import os
import json
from datetime import datetime

now = datetime.now()
dt_string = now.strftime("%d-%m-%Y_%H:%M:%S")

cred = credentials.Certificate("vinterprosjekt-it2-firebase-adminsdk-wi0bg-f0efd8881e.json")
firebase_admin.initialize_app(cred, {'databaseURL':"https://vinterprosjekt-it2-default-rtdb.europe-west1.firebasedatabase.app/"})

ref = db.reference("/bazaar")

with open("bazaar_info.json", "r") as f:
	file_contents = json.load(f)
# ref.set(file_contents)

ref.child(dt_string).set(file_contents)

os.remove("bazaar_info.json")

ref = db.reference("/auctions")

with open("auctions_info.json", "r") as f:
	file_contents = json.load(f)
# ref.set(file_contents)

ref.child(dt_string).set(file_contents)

os.remove("auctions_info.json")

# data_push = ref.push().set(file_contents)

# post_id = data_push.key


from __future__ import absolute_import, unicode_literals

import json
import logging

from django.conf import settings
from django.db.models import Q, Count
from django.core.management.base import BaseCommand

from muses.collection.models import Item
from muses.naive_classification.helpers_os import predict_image_path_dict

LOGGER = logging.getLogger(__name__)

class Command(BaseCommand):
    """Classify."""

    help = "Classify items with our AI."

    requires_system_checks = False

    def add_arguments(self, parser):
        parser.add_argument('--update-existing',
                            action='store_true',
                            dest='update_existing',
                            default=False,
                            help="Update existing classifications.")

    def handle(self, *args, **options):
        """Handle.

        :param args:
        :param options:
        :return:
        """
        update_existing = bool(options['update_existing'])

        filters = []
        if not update_existing:
            for field in ['classified_as']:
                filters.append(
                    Q(**{"{}__isnull".format(field): True})
                    | Q(**{"{}__exact".format(field): ''})
                )

        items = Item \
            .objects \
            .filter(*filters) \
            .prefetch_related('images') \
            .annotate(num_images=Count('images')) \
            .filter(num_images__gt=0)

        for item in items:
            paths = []
            for image in item.images.all():
                try:
                    paths.append(image.image.path)
                except Exception as err:
                    LOGGER.warning(err)

            conf = settings \
                .MUSES_CONFIG['classification']['naive_classification']
            model_path = conf['model_path']

            try:
                classification = predict_image_path_dict(
                    paths,
                    model_path=model_path
                )
            except Exception as err:
                LOGGER.warning(err)
                continue

            top_results = list(classification.items())[:5]
            if top_results:
                try:
                    item.classified_as = top_results
                    item.save()
                except Exception as err:
                    pass


"""Provides a ChatBot UI for a Github Repository. Powered by Llama Index and Panel"""
import os
import pickle
from pathlib import Path

import nest_asyncio
import panel as pn
import param
from llama_index import VectorStoreIndex, download_loader

from llama_hub.github_repo import GithubClient, GithubRepositoryReader

# needed because both Panel and GithubRepositoryReader starts up the ioloop
nest_asyncio.apply()

CACHE_PATH = Path(".cache/panel_chatbot")
CACHE_PATH.mkdir(parents=True, exist_ok=True)

CHAT_GPT_LOGO = "https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/ChatGPT_logo.svg/512px-ChatGPT_logo.svg.png"
CHAT_GPT_URL = "https://chat.openai.com/"
LLAMA_INDEX_LOGO = (
    "https://cdn-images-1.medium.com/max/280/1*_mrG8FG_LiD23x0-mEtUkw@2x.jpeg"
)
PANEL_LOGO = {
    "default": "https://panel.holoviz.org/_static/logo_horizontal_light_theme.png",
    "dark": "https://panel.holoviz.org/_static/logo_horizontal_dark_theme.png",
}

GITHUB_LOGO = "https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png"
GITHUB_URL = "https://github.com/"
LLAMA_INDEX_URL = "https://www.llamaindex.ai/"
PANEL_URL = "https://panel.holoviz.org/index.html"
GITHUB_COPILOT_LOGO = (
    "https://plugins.jetbrains.com/files/17718/447537/icon/pluginIcon.svg"
)

INDEX_NOT_LOADED = "No repository loaded"
INDEX_LOADED = "Repository loaded"
LOADING_EXISTING_DOCS = "Loading existing docs"
LOADING_NEW_DOCS = "Downloading documents"
LOADING_EXISTING_INDEX = "Loading existing index"
LOADING_NEW_INDEX = "Creating index"
CUTE_LLAMA = "https://raw.githubusercontent.com/run-llama/llama-hub/main/llama_hub/llama_packs/panel_chatbot/llama_by_sophia_yang.png"
CUTE_LLAMA_URL = "https://x.com/sophiamyang/status/1729810715467252080?s=20"

pn.chat.ChatMessage.default_avatars.update(
    {
        "assistant": GITHUB_COPILOT_LOGO,
        "user": "ðŸ¦™",
    }
)
pn.chat.ChatMessage.show_reaction_icons = False

ACCENT = "#ec4899"

CSS_FIXES_TO_BE_UPSTREAMED_TO_PANEL = """
#sidebar {
    padding-left: 5px !important;
    background-color: var(--panel-surface-color);
}
.pn-wrapper {
    height: calc( 100vh - 150px);
}
.bk-active.bk-btn-primary {border-color: var(--accent-fill-active)}
.bk-btn-primary:hover {border-color: var(--accent-fill-hover)}
.bk-btn-primary {border-color: var(--accent-fill-rest)}
a {color: var(--accent-fill-rest) !important;}
a:hover {color: var(--accent-fill-hover) !important;}
"""

def _split_and_clean(cstext):
    return cstext.split(",")

class IndexLoader(pn.viewable.Viewer):
    """The IndexLoader enables the user to interactively create a VectorStoreIndex from a
    github repository of choice"""

    value: VectorStoreIndex = param.ClassSelector(class_=VectorStoreIndex)

    status = param.String(constant=True, doc="A status message")

    owner: str = param.String(
        default="holoviz", doc="The repository owner. For example 'holoviz'"
    )
    repo: str = param.String(
        default="panel", doc="The repository name. For example 'panel'"
    )
    filter_directories: str = param.String(
        default="examples,docs,panel",
        label="Folders",
        doc="A comma separated list of folders to include. For example 'examples,docs,panel'",
    )
    filter_file_extensions: str = param.String(
        default=".py,.md,.ipynb",
        label="File Extensions",
        doc="A comma separated list of file extensions to include. For example '.py,.md,.ipynb'",
    )

    _load = param.Event(
        label="LOAD",
        doc="Loads the repository index from the cache if it exists and otherwise from scratch",
    )
    _reload = param.Event(
        default=False,
        label="RELOAD ALL",
        doc="Loads the repository index from scratch",
    )

    def __init__(self):
        super().__init__()

        if self.index_exists:
            pn.state.execute(self.load)
        else:
            self._update_status(INDEX_NOT_LOADED)

        self._layout = pn.Column(
            self.param.owner,
            self.param.repo,
            self.param.filter_directories,
            self.param.filter_file_extensions,
            pn.pane.HTML(self.github_url),
            pn.widgets.Button.from_param(
                self.param._load,
                button_type="primary",
                disabled=self._is_loading,
                loading=self._is_loading,
            ),
            pn.widgets.Button.from_param(
                self.param._reload,
                button_type="primary",
                button_style="outline",
                disabled=self._is_loading,
                loading=self._is_loading,
            ),
            pn.pane.Markdown("### Status", margin=(3, 5)),
            pn.pane.Str(self.param.status),
        )

    def __panel__(self):
        return self._layout

    @property
    def _unique_id(self):
        uid = (
            self.owner
            + self.repo
            + self.filter_directories
            + self.filter_file_extensions
        )
        uid = uid.replace(",", "").replace(".", "")
        return uid

    @property
    def _cached_docs_path(self):
        return CACHE_PATH / f"docs_{self._unique_id}.pickle"

    @property
    def _cached_index_path(self):
        return CACHE_PATH / f"index_{self._unique_id}.pickle"

    async def _download_docs(self):
        download_loader("GithubRepositoryReader")

        github_client = GithubClient(os.getenv("GITHUB_TOKEN"))

        filter_directories = _split_and_clean(self.filter_directories)
        filter_file_extensions = _split_and_clean(self.filter_file_extensions)

        loader = GithubRepositoryReader(
            github_client,
            owner=self.owner,
            repo=self.repo,
            filter_directories=(
                filter_directories,
                GithubRepositoryReader.FilterType.INCLUDE,
            ),
            filter_file_extensions=(
                filter_file_extensions,
                GithubRepositoryReader.FilterType.INCLUDE,
            ),
            verbose=True,
            concurrent_requests=10,
        )
        return loader.load_data(branch="main")

    async def _get_docs(self):
        docs_path = self._cached_docs_path
        index_path = self._cached_index_path

        if docs_path.exists():
            self._update_status(LOADING_EXISTING_DOCS)
            with docs_path.open("rb") as f:
                return pickle.load(f)

        self._update_status(LOADING_NEW_DOCS)
        docs = await self._download_docs()

        with docs_path.open("wb") as f:
            pickle.dump(docs, f, pickle.HIGHEST_PROTOCOL)

        if index_path.exists():
            index_path.unlink()

        return docs

    async def _create_index(self, docs):
        return VectorStoreIndex.from_documents(docs, use_async=True)

    async def _get_index(self, index):
        index_path = self._cached_index_path

        if index_path.exists():
            self._update_status(LOADING_EXISTING_INDEX)
            with index_path.open("rb") as f:
                return pickle.load(f)

        self._update_status(LOADING_NEW_INDEX)
        index = await self._create_index(index)

        with index_path.open("wb") as f:
            pickle.dump(index, f, pickle.HIGHEST_PROTOCOL)
        return index

    @param.depends("status")
    def _is_loading(self):
        return self.status not in [INDEX_LOADED, INDEX_NOT_LOADED]

    @param.depends("status")
    def _is_not_loading(self):
        return self.status in [INDEX_LOADED, INDEX_NOT_LOADED]

    @param.depends("_load", watch=True)
    async def load(self):
        """Loads the repository index either from the cache or by downloading from
        the repository"""
        self._update_status("Loading ...")
        self.value = None

        docs = await self._get_docs()
        self.value = await self._get_index(docs)
        self._update_status(INDEX_LOADED)

    @param.depends("_reload", watch=True)
    async def reload(self):
        self._update_status("Deleteing cached index ...")
        if self._cached_docs_path.exists():
            self._cached_docs_path.unlink()
        if self._cached_index_path.exists():
            self._cached_index_path.unlink()

        await self.load()

    def _update_status(self, text):
        with param.edit_constant(self):
            self.status = text
        print(text)

    @param.depends("owner", "repo")
    def github_url(self):
        """Returns a html string with a link to the github repository"""
        text = f"{self.owner}/{self.repo}"
        href = f"https://github.com/{text}"
        return f"{text}"

    @property
    def index_exists(self):
        """Returns True if the index already exists"""
        return self._cached_docs_path.exists() and self._cached_index_path.exists()

def powered_by():
    """Returns a component describing the frameworks powering the chat ui"""
    params = {"height": 40, "sizing_mode": "fixed", "margin": (0, 10)}
    return pn.Column(
        pn.pane.Markdown("### AI Powered By", margin=(10, 5, 10, 0)),
        pn.Row(
            pn.pane.Image(LLAMA_INDEX_LOGO, link_url=LLAMA_INDEX_URL, **params),
            pn.pane.Image(CHAT_GPT_LOGO, link_url=CHAT_GPT_URL, **params),
            pn.pane.Image(PANEL_LOGO[pn.config.theme], link_url=PANEL_URL, **params),
            align="center",
        ),
    )

async def chat_component(index: VectorStoreIndex, index_loader: IndexLoader):
    """Returns the chat component powering the main area of the application"""
    if not index:
        return pn.Column(
            pn.chat.ChatMessage(
                "You are a now a *GitHub Repository assistant*.",
                user="System",
            ),
            pn.chat.ChatMessage(
                "Please **load a GitHub Repository** to start chatting with me. This can take from seconds to minutes!",
                user="Assistant",
            ),
        )

    chat_engine = index.as_chat_engine(chat_mode="context", verbose=True)

    async def generate_response(contents, user, instance):
        response = await chat_engine.astream_chat(contents)
        text = ""
        async for token in response.async_response_gen():
            text += token
            yield text

    chat_interface = pn.chat.ChatInterface(
        callback=generate_response,
        sizing_mode="stretch_both",
    )
    chat_interface.send(
        pn.chat.ChatMessage(
            "You are a now a *GitHub Repository Assistant*.", user="System"
        ),
        respond=False,
    )
    chat_interface.send(
        pn.chat.ChatMessage(
            f"Hello! you can ask me anything about {index_loader.github_url()}.",
            user="Assistant",
        ),
        respond=False,
    )
    return chat_interface

def settings_components(index_loader: IndexLoader):
    """Returns a list of the components to add to the sidebar"""
    return [
        pn.pane.Image(
            CUTE_LLAMA,
            height=250,
            align="center",
            margin=(10, 5, 25, 5),
            link_url=CUTE_LLAMA_URL,
        ),
        "## Github Repository",
        index_loader,
        powered_by(),
    ]

def create_chat_ui():
    """Returns the Chat UI"""
    pn.extension(
        sizing_mode="stretch_width", raw_css=[CSS_FIXES_TO_BE_UPSTREAMED_TO_PANEL]
    )

    index_loader = IndexLoader()

    pn.state.location.sync(
        index_loader,
        parameters={
            "owner": "owner",
            "repo": "repo",
            "filter_directories": "folders",
            "filter_file_extensions": "file_extensions",
        },
    )

    bound_chat_interface = pn.bind(
        chat_component, index=index_loader.param.value, index_loader=index_loader
    )

    return pn.template.FastListTemplate(
        title="Chat with GitHub",
        sidebar=settings_components(index_loader),
        main=[bound_chat_interface],
        accent=ACCENT,
        main_max_width="1000px",
        main_layout=None,
    )

if pn.state.served:
    create_chat_ui().servable()


# - *- coding: utf- 8 - *-
import time

from pepper.robot import Pepper
try:
    import urllib
except:
    import urllib.request as urllib
import base64
import json
from PIL import Image
import random

def uploadPhotoToWeb(photo):
    """we need to upload photo to web as we (me) are not able to open it from local folder"""
    f = open(photo, "rb")  # open our image file as read only in binary mode
    image_data = f.read()  # read in our image file
    b64_image = base64.standard_b64encode(image_data)
    client_id = "af482612ae6d1c1"  # this the id which we've got after registrating the app on imgur
    headers = {'Authorization': 'Client-ID ' + client_id}
    data = {'image': b64_image, 'title': 'test'}
    request = urllib.Request(url="https://api.imgur.com/3/upload.json", data=urllib.urlencode(data),
                              headers=headers)
    response = urllib.urlopen(request).read()
    parse = json.loads(response)
    return parse['data']['link'] #returns a url of the photo

def getRandName():
    """returns a random name for the picture in order not to replace the old photo"""
    randNum = random.randint(0, 1000)
    return "demoPictures/photo" + str(randNum) + ".png"

class PepperDemo:
    def __init__(self, ip_address, port=9559):
        self.robot = None
        self.robot = Pepper(ip_address, port)
        self.robot.set_czech_language()
        self.photoName = None
        self.greetings = ["Good afternoon", "Hello", "Hi", "Hello everobody", "Welcome"]
        self.asks = ["May I photograph you?","May I take your picture?", "Do you want to make your picture?"]

    def wantToTakePic(self):
        """recognise answer with google speech reco"""
        answers = {"no": ["no", "no way", "not", "no no", " i don't", "i dont know", "not today", "later", "tommorow"],
                   "yes": ["yes", "definitely", "yep", "ok", "okey dokey", "sure", "all yes", "you must",
                          "absolutely", "i want", "i think so", "i agree", "if you want", "if you insist", "probably", "maybe",
                          "yes sir"]}
        recorded = self.robot.recognize_google(lang="en-US")
        answer = self.getAnswer(answers, recorded)
        if answer == "no":
            return False
        elif answer == "yes":
            return True
        else:
            return None

    def getAnswer(self, dic, recorded):
        """looks for a recorded answer in a dictionar"""
        for x in dic.keys():
            if dic[x] in recorded.lower():
                return x
        return None

    def welcomeAndAsk(self):
        self.robot.say(random.choice(self.greetings))
        self.robot.greet()
        self.robot.say(random.choice(self.asks))

    def takePicture(self):
        self.robot.subscribe_camera("camera_top", 2, 30)
        img = self.robot.get_camera_frame(show=False)
        self.robot.unsubscribe_camera()
        self.robot.play_sound("/home/nao/camera1.ogg")
        im = Image.fromarray(img)
        self.photoName = getRandName()
        im.save(self.photoName)

    def showPicture(self):
        link = uploadPhotoToWeb(self.photoName)
        self.robot.show_image(link)
        time.sleep(5)
        self.robot.reset_tablet()

    def recogniseAnswerAndDecide(self):
        isTakePic = self.wantToTakePic()
        if isTakePic:
            self.robot.say("Perfect. On your marks. 3, 2, 1 .")
            self.takePicture()
            self.showPicture()
        elif isTakePic is None:
            self.robot.say("Sorry, I did not understand you. Please repeat.")
            self.recogniseAnswerAndDecide()
        else:
            self.robot.say("Maybe next time")

    def dealWithRecoErrors(self):
        """there is a modifiable grammar error sometimes occurred.
        In order to deal with it you should change language to english and back"""
        self.robot.set_english_language()
        self.robot.set_czech_language()

    def run(self):
        self.dealWithRecoErrors()
        self.welcomeAndAsk()
        self.recogniseAnswerAndDecide()

if __name__ == "__main__":
    pepperDemo = PepperDemo("10.37.1.232")
    pepperDemo.run()


from bson import ObjectId
from dataclasses import field
from typing import Optional

from aenum import Enum
from isodate import parse_duration

from extensions.authorization.models.authorized_user import AuthorizedUser
from extensions.authorization.models.invitation import Invitation, InvitationType
from extensions.authorization.models.role.default_roles import DefaultRoles
from extensions.authorization.models.role.role import RoleName
from extensions.authorization.validators import (
    check_role_id_valid_for_organization,
    is_common_role,
)
from extensions.common.sort import SortField
from sdk.auth.use_case.auth_request_objects import BaseAuthRequestObject
from sdk.common.exceptions.exceptions import InvalidRequestException, PermissionDenied
from sdk.common.localization.utils import Language
from sdk.common.usecase.request_object import RequestObject
from sdk.common.utils import inject
from sdk.common.utils.convertible import (
    convertibleclass,
    meta,
    required_field,
    positive_integer_field,
    default_field,
)
from sdk.common.utils.inject import autoparams
from sdk.common.utils.validators import (
    validate_id,
    validate_email_list,
    validate_object_id,
    validate_object_ids,
    must_be_present,
    must_not_be_present,
    validate_email,
    incorrect_language_to_default,
    must_be_at_least_one_of,
    not_empty,
)
from sdk.phoenix.config.server_config import Client

INVITATION_PERMISSIONS_PER_ROLE: dict[str, list[str]] = {
    RoleName.ADMINISTRATOR: [
        RoleName.ADMINISTRATOR,
        RoleName.CLINICIAN,
        RoleName.SUPERVISOR,
        RoleName.SUPPORT,
        RoleName.USER,
    ],
    RoleName.CLINICIAN: [RoleName.USER, RoleName.PROXY],
}

def validate_role_id(field_value: str) -> bool:
    if validate_default_role_id(field_value):
        return True
    if validate_id(field_value):
        return True
    return False

@autoparams("default_roles")
def validate_default_role_id(value: str, default_roles: DefaultRoles) -> bool:
    return value in default_roles

@convertibleclass
class SendInvitationRequestObject(RequestObject):
    INVITATION = "invitation"
    CLIENT = "client"
    SENDER = "sender"
    LANGUAGE = "language"
    EXTRA_INFO = "extraInfo"

    invitation: Invitation = required_field()
    client: Client = required_field()
    sender: AuthorizedUser = required_field()
    language: str = field(
        default=Language.EN, metadata=meta(value_to_field=incorrect_language_to_default)
    )
    extraInfo: dict = default_field()

@convertibleclass
class SendInvitationsRequestObject(BaseAuthRequestObject):
    DEPLOYMENT_IDS = "deploymentIds"
    ORGANIZATION_ID = "organizationId"
    EMAILS = "emails"
    ROLE_ID = "roleId"
    PATIENT_ID = "patientId"
    EXPIRES_IN = "expiresIn"
    SUBMITTER = "submitter"

    emails: list[str] = required_field(metadata=meta(validate_email_list))
    roleId: str = required_field(metadata=meta(validate_role_id))
    organizationId: str = default_field(metadata=meta(validate_object_id))
    deploymentIds: list[str] = default_field(metadata=meta(validate_object_ids))
    patientId: str = default_field(metadata=meta(validate_object_id))
    expiresIn: str = field(default="P1W", metadata=meta(parse_duration))
    submitter: AuthorizedUser = required_field()

    @classmethod
    def validate(cls, instance):
        super().validate(instance)
        is_org_role = check_role_id_valid_for_organization(
            instance.roleId, instance.organizationId
        )
        is_proxy_role = instance.roleId == RoleName.PROXY
        is_common_role = instance.roleId in RoleName.common_roles()
        if is_common_role:
            must_be_at_least_one_of(
                organizationId=instance.organizationId,
                deploymentIds=instance.deploymentIds,
            )
            must_not_be_present(patientId=instance.patientId)

            if instance.deploymentIds and len(instance.deploymentIds) > 1:
                must_be_present(organizationId=instance.organizationId)
        elif is_org_role:
            must_be_present(organizationId=instance.organizationId)
            must_not_be_present(deploymentIds=instance.deploymentIds)
            must_not_be_present(patientId=instance.patientId)
        elif is_proxy_role:
            must_be_present(patientId=instance.patientId)
            must_not_be_present(deploymentIds=instance.deploymentIds)
            must_not_be_present(organizationId=instance.organizationId)
        else:
            must_not_be_present(patientId=instance.patientId)
            must_be_present(deploymentIds=instance.deploymentIds)
            if len(instance.deploymentIds) == 0:
                msg = f"Must be invited to at least one deployment"
                raise InvalidRequestException(msg)

            multiple_deployment_role = (
                instance.roleId in cls.multiple_deployment_roles()
            )

            if multiple_deployment_role and len(instance.deploymentIds) > 1:
                must_be_present(organizationId=instance.organizationId)

            if not multiple_deployment_role and len(instance.deploymentIds) > 1:
                msg = f"Role {instance.roleId} can only be invited to one deployment"
                raise InvalidRequestException(msg)

    def check_permission(self, submitter: AuthorizedUser):
        submitter_role = submitter.get_role()

        if submitter.is_super_admin():
            return

        if is_common_role(submitter_role.id):
            if not self.roleId == RoleName.PROXY:
                if not submitter.role_assignment.is_org():
                    if not self.deploymentIds:
                        raise PermissionDenied
                    else:
                        self.validate_resource_access(
                            "deployment", self.deploymentIds, submitter
                        )
                else:
                    if self.deploymentIds:
                        self.validate_resource_access(
                            "deployment", self.deploymentIds, submitter
                        )
                    elif self.organizationId:
                        self.validate_resource_access(
                            "organization", [self.organizationId], submitter
                        )

            allowed_roles = INVITATION_PERMISSIONS_PER_ROLE.get(submitter_role.id) or []
            if ObjectId.is_valid(self.roleId):
                allowed_roles.append(self.roleId)
            if self.roleId not in allowed_roles:
                raise PermissionDenied

    @staticmethod
    def validate_resource_access(
        resource_name: str, resources: list[str], submitter: AuthorizedUser
    ):
        allowed_resources = []

        if resource_name == "organization":
            allowed_resources = submitter.organization_ids()
        elif resource_name == "deployment":
            allowed_resources = submitter.deployment_ids()

        if not all(resource_id in allowed_resources for resource_id in resources):
            raise PermissionDenied

    @staticmethod
    def multiple_deployment_roles():
        org_keys = inject.instance(DefaultRoles).organization.keys()
        return (set(org_keys) - set(RoleName.org_roles())).union(
            set(RoleName.common_roles())
        )

    @property
    def deployment_id(self) -> Optional[str]:
        if self.deploymentIds:
            return self.deploymentIds[0]

@convertibleclass
class ResendInvitationsRequestObject(BaseAuthRequestObject):
    INVITATION_CODE = "invitationCode"
    EMAIL = "email"

    email: str = required_field(metadata=meta(validate_email))
    invitationCode: str = required_field()

@convertibleclass
class ResendInvitationsListRequestObject(BaseAuthRequestObject):
    INVITATIONS_LIST = "invitationsList"

    @convertibleclass
    class InvitationItem:
        INVITATION_CODE = "invitationCode"
        EMAIL = "email"

        email: str = required_field(metadata=meta(validate_email))
        invitationCode: str = required_field(metadata=meta(not_empty))

    invitationsList: list[InvitationItem] = required_field(metadata=meta(not_empty))

@convertibleclass
class GetInvitationLinkRequestObject(BaseAuthRequestObject):
    DEPLOYMENT_ID = "deploymentId"
    ROLE_ID = "roleId"
    EXPIRES_IN = "expiresIn"
    RETRIEVE_SHORTENED = "retrieveShortened"
    SENDER_ID = "senderId"

    deploymentId: str = required_field()
    roleId: str = required_field()
    expiresIn: str = field(default="P1W", metadata=meta(parse_duration))
    retrieveShortened: bool = field(default=False)
    senderId: str = required_field(metadata=meta(validate_object_id))

@convertibleclass
class DeleteInvitationRequestObject(RequestObject):
    INVITATION_ID = "invitationId"

    invitationId: str = required_field(metadata=meta(validate_object_id))

@convertibleclass
class DeleteInvitationsListRequestObject(RequestObject):
    INVITATION_ID_LIST = "invitationIdList"
    INVITATION_TYPE = "invitationType"

    invitationIdList: list[str] = required_field(
        metadata=meta(lambda x: all(map(validate_object_id, x)))
    )
    invitationType: InvitationType = field(default=InvitationType.PERSONAL)

@convertibleclass
class RetrieveInvitationsRequestObject(BaseAuthRequestObject):
    EMAIL = "email"
    ROLE_TYPE = "roleType"
    SKIP = "skip"
    LIMIT = "limit"
    SUBMITTER = "submitter"
    INVITATION_TYPE = "invitationType"
    SORT_FIELDS = "sortFields"

    class RoleType(Enum):
        MANAGER = "Manager"
        USER = "User"

    email: str = default_field()
    roleType: RoleType = required_field()
    skip: int = positive_integer_field(default=None, metadata=meta(required=True))
    limit: int = positive_integer_field(default=None, metadata=meta(required=True))
    submitter: AuthorizedUser = required_field()
    invitationType: InvitationType = default_field()
    sortFields: list[SortField] = default_field()

    def post_init(self):
        if not self.sortFields:
            self.sortFields = [
                SortField.from_dict(
                    {
                        SortField.FIELD: Invitation.CREATE_DATE_TIME,
                        SortField.DIRECTION: SortField.Direction.DESC.value,
                    }
                )
            ]

@convertibleclass
class InvitationValidityRequestObject(RequestObject):
    INVITATION_CODE = "invitationCode"

    invitationCode: str = required_field()


# -*- coding:utf-8 -*-
import requests, os
from flask import render_template, request, flash, redirect, url_for, send_from_directory, send_file
from flask_login import current_user, login_required
from oauth2client.service_account import ServiceAccountCredentials
from pandas import DataFrame
from pydrive.auth import GoogleAuth
from reportlab.lib import styles
from reportlab.lib.enums import TA_RIGHT, TA_CENTER
from reportlab.lib.pagesizes import letter
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.utils import ImageReader
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.platypus import SimpleDocTemplate, Paragraph, Image, TableStyle, Table, Spacer
from sqlalchemy import cast, Date
from werkzeug.utils import secure_filename
from . import purchase_tracker_bp as purchase_tracker
from .forms import *
from datetime import datetime
from pytz import timezone
from pydrive.drive import GoogleDrive
from .models import PurchaseTrackerAccount, PurchaseTrackerForm
from flask_mail import Message
from ..main import mail
from ..roles import finance_procurement_permission

# Upload images for Google Drive

FOLDER_ID = "1JYkU2kRvbvGnmpQ1Tb-TcQS-vWQKbXvy"

json_keyfile = requests.get(os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')).json()

bangkok = timezone('Asia/Bangkok')

ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'}

@purchase_tracker.route('/official/')
@login_required
def landing_page():
    return render_template('purchase_tracker/first_page.html')

@purchase_tracker.route('/personnel/personnel_index')
def staff_index():
    return render_template('purchase_tracker/personnel/personnel_index.html')

@purchase_tracker.route('/personnel/personnel_index/e-form/method/select/<int:account_id>')
def select_form(account_id):
    account = PurchaseTrackerAccount.query.get(account_id)
    return render_template('purchase_tracker/personnel/alternative_form.html', account=account)

@purchase_tracker.route('/main')
def index():
    return render_template('purchase_tracker/index.html')

@purchase_tracker.route('/account/create', methods=['GET', 'POST'])
@login_required
def add_account():
    form = CreateAccountForm()
    if request.method == 'POST':
        filename = ''
        account = PurchaseTrackerAccount()
        form.populate_obj(account)
        account.creation_date = bangkok.localize(datetime.now())
        account.staff = current_user
        drive = initialize_gdrive()
        if form.upload.data:
            if not filename or (form.upload.data.filename != filename):
                upfile = form.upload.data
                filename = secure_filename(upfile.filename)
                upfile.save(filename)
                file_drive = drive.CreateFile({'title': filename,
                                               'parents': [{'id': FOLDER_ID, "kind": "drive#fileLink"}]})
                file_drive.SetContentFile(filename)
                try:
                    file_drive.Upload()
                    permission = file_drive.InsertPermission({'type': 'anyone',
                                                              'value': 'anyone',
                                                              'role': 'reader'})
                except:
                    flash('Failed to upload the attached file to the Google drive.', 'danger')
                else:
                    flash('The attached file has been uploaded to the Google drive', 'success')
                    account.url = file_drive['id']

        db.session.add(account)
        db.session.commit()
        flash(u'บันทึกข้อมูลสำเร็จ.', 'success')
        return render_template('purchase_tracker/personnel/personnel_index.html')
        # Check Error
    else:
        for er in form.errors:
            flash(er, 'danger')
    return render_template('purchase_tracker/create_account.html', form=form)

def initialize_gdrive():
    gauth = GoogleAuth()
    scopes = ['https://www.googleapis.com/auth/drive']
    gauth.credentials = ServiceAccountCredentials.from_json_keyfile_dict(json_keyfile, scopes)
    return GoogleDrive(gauth)

@purchase_tracker.route('/track/')
@purchase_tracker.route('/track/<int:account_id>', methods=['GET'])
def track(account_id=None):
    list_type = request.args.get('list_type')
    if list_type == "myAccount" or list_type is None:
        accounts = PurchaseTrackerAccount.query.filter_by(staff_id=current_user.id).all()
    elif list_type == "ourAccount":
        org = current_user.personal_info.org
        accounts = [account for account in PurchaseTrackerAccount.query.all()
                    if account.staff.personal_info.org == org]
    return render_template('purchase_tracker/tracking.html',
                           account_id=account_id, accounts=accounts, list_type=list_type)

@purchase_tracker.route('/track/<int:account_id>/view')
def view_info_track(account_id=None):
    from sqlalchemy import desc
    if account_id:
        account = PurchaseTrackerAccount.query.get(account_id)
        # better make use of the relationship!
        activities = [a.to_list() for a in account.records.all()]
    else:
        flash(u'ข้อมูลจะปรากฎเมื่อหน่วยงานคลังและพัสดุอัพเดตเรียบร้อย', 'warning')
        activities = []
    # activities = [a.to_list() for a in PurchaseTrackerStatus.query.filter_by(account_id=account_id)
    #     .order_by(PurchaseTrackerStatus.start_date)]
    if not activities:
        default_date = datetime.now().isoformat()
    else:
        default_date = activities[-1][3]
    return render_template('purchase_tracker/view_info_track.html',
                           account_id=account_id, account=account, desc=desc,
                           PurchaseTrackerStatus=PurchaseTrackerStatus,
                           activities=activities, default_date=default_date)

@purchase_tracker.route('/account/<int:account_id>/edit', methods=['GET', 'POST'])
@login_required
def edit_account(account_id):
    account = PurchaseTrackerAccount.query.get(account_id)
    form = CreateAccountForm(obj=account)
    if request.method == 'POST':
        filename = ''
        form.populate_obj(account)
        account.creation_date = bangkok.localize(datetime.now())
        account.staff = current_user
        drive = initialize_gdrive()
        if form.upload.data:
            if not filename or (form.upload.data.filename != filename):
                upfile = form.upload.data
                filename = secure_filename(upfile.filename)
                upfile.save(filename)
                file_drive = drive.CreateFile({'title': filename,
                                               'parents': [{'id': FOLDER_ID, "kind": "drive#fileLink"}]})
                file_drive.SetContentFile(filename)
                try:
                    file_drive.Upload()
                    permission = file_drive.InsertPermission({'type': 'anyone',
                                                              'value': 'anyone',
                                                              'role': 'reader'})
                except:
                    flash('Failed to upload the attached file to the Google drive.', 'danger')
                else:
                    flash('The attached file has been uploaded to the Google drive', 'success')
                    purchase_tracker.url = file_drive['id']

        db.session.add(account)
        db.session.commit()
        flash(u'บันทึกข้อมูลสำเร็จ.', 'success')
        return redirect(url_for('purchase_tracker.track'))
        # Check Error
    else:
        for er in form.errors:
            flash(er, 'danger')
    return render_template('purchase_tracker/edit_account.html', form=form, account_id=account_id)

@purchase_tracker.route('/accounts/<int:account_id>/cancel', methods=['GET'])
@login_required
def cancel_account(account_id):
    account = PurchaseTrackerAccount.query.get(account_id)
    if not account.cancelled_datetime:
        account.cancelled_datetime = datetime.now(tz=bangkok)
        account.cancelled_by = current_user
        db.session.add(account)
        db.session.commit()
        flash(u'ยกเลิกบัญชีเรียบร้อยแล้ว', 'success')
    else:
        flash(u'บัญชีนี้ถูกยุติการดำเนินการแล้ว', 'warning')
    next = request.args.get('next')
    if next:
        return redirect(next)
    return redirect(url_for('purchase_tracker.track', account_id=account_id))

@purchase_tracker.route('/accounts/<int:account_id>/close', methods=['GET'])
@login_required
def close_account(account_id):
    account = PurchaseTrackerAccount.query.get(account_id)
    if not account.end_datetime:
        account.end_datetime = datetime.now(tz=bangkok)
        db.session.add(account)
        db.session.commit()
        flash(u'ปิดบัญชีเรียบร้อยแล้ว', 'success')
    else:
        flash(u'บัญชีนี้ถูกปิดการดำเนินการแล้ว', 'warning')
    return redirect(url_for('purchase_tracker.update_status', account_id=account_id))

@purchase_tracker.route('/supplies/')
@finance_procurement_permission.require()
def supplies():
    from sqlalchemy import desc
    accounts = PurchaseTrackerAccount.query.all()
    return render_template('purchase_tracker/procedure_supplies.html',
                           accounts=accounts,
                           desc=desc,
                           PurchaseTrackerStatus=PurchaseTrackerStatus)

@purchase_tracker.route('/description')
def description():
    return render_template('purchase_tracker/description.html')

@purchase_tracker.route('/contact')
def contact():
    return render_template('purchase_tracker/contact_us.html')

def send_mail(recp, title, message):
    message = Message(subject=title, body=message, recipients=recp)
    mail.send(message)

@purchase_tracker.route('/account/<int:account_id>/update', methods=['GET', 'POST'])
@finance_procurement_permission.require()
@login_required
def update_status(account_id):
    form = StatusForm()
    account = PurchaseTrackerAccount.query.get(account_id)
    if request.method == 'POST':
        if form.validate_on_submit():
            status = PurchaseTrackerStatus()
            form.populate_obj(status)
            status.account_id = account_id
            status.status_date = bangkok.localize(datetime.now())
            status.creation_date = bangkok.localize(datetime.now())
            status.cancel_datetime = bangkok.localize(datetime.now())
            status.update_datetime = bangkok.localize(datetime.now())
            status.staff = current_user
            if not form.other_activity.data and not form.activity.data:
                flash(u'กรุณาเลือกหัวข้อกิจกรรมหรือใส่กิจกรรมอื่นๆ.', 'danger')
                return redirect(
                    url_for('purchase_tracker.update_status', account_id=account_id, form=form, account=account))
            db.session.add(status)
            db.session.commit()
            title = u'แจ้งเตือนการปรับเปลี่ยนสถานะการจัดซื้อพัสดุและครุภัณฑ์หมายเลข {}'.format(status.account.number)
            message = u'เรียน {}\n\nสถานะการจัดซื้อพัสดุและครุภัณฑ์หมายเลข {} คือ {}' \
                .format(current_user.personal_info.fullname, status.account.number,
                        status.other_activity or status.activity.activity)
            message += u'\n\n======================================================'
            message += u'\nอีเมลนี้ส่งโดยระบบอัตโนมัติ กรุณาอย่าตอบกลับ ' \
                       u'หากมีปัญหาใดๆเกี่ยวกับเว็บไซต์กรุณาติดต่อหน่วยข้อมูลและสารสนเทศ '
            message += u'\nThis email was sent by an automated system. Please do not reply.' \
                       u' If you have any problem about website, please contact the IT unit.'
            send_mail([u'{}@mahidol.ac.th'.format(account.staff.email)], title, message)
            flash(u'อัพเดตข้อมูลเรียบร้อย', 'success')
            form.activity.data = ""
            form.other_activity.data = ""
            form.comment.data = ""
        # Check Error
        else:
            flash(form.errors, 'danger')

    activities = [a.to_list() for a in PurchaseTrackerStatus.query.filter_by(account_id=account_id)
        .order_by(PurchaseTrackerStatus.start_date)]
    if not activities:
        default_date = datetime.now().isoformat()
    else:
        default_date = activities[-1][3]
    return render_template('purchase_tracker/update_record.html',
                           account_id=account_id, form=form, activities=activities, account=account,
                           default_date=default_date)

@purchase_tracker.route('/account/<int:account_id>/status/<int:status_id>/edit', methods=['GET', 'POST'])
@finance_procurement_permission.require()
@login_required
def edit_update_status(account_id, status_id):
    status = PurchaseTrackerStatus.query.get(status_id)
    form = StatusForm(obj=status)
    if request.method == 'POST':
        if form.validate_on_submit():
            form.populate_obj(status)
            status.account_id = account_id
            status.status_date = bangkok.localize(datetime.now())
            status.creation_date = bangkok.localize(datetime.now())
            status.cancel_datetime = bangkok.localize(datetime.now())
            status.update_datetime = bangkok.localize(datetime.now())
            status.staff = current_user
            db.session.add(status)
            db.session.commit()
            title = u'แจ้งเตือนการแก้ไขปรับเปลี่ยนสถานะการจัดซื้อพัสดุและครุภัณฑ์หมายเลข {}'.format(
                status.account.number)
            message = u'เรียน {}\n\nสถานะการจัดซื้อพัสดุและครุภัณฑ์หมายเลข {} คือ {}' \
                .format(current_user.personal_info.fullname, status.account.number,
                        status.other_activity or status.activity.activity)
            message += u'\n\n======================================================'
            message += u'\nอีเมลนี้ส่งโดยระบบอัตโนมัติ กรุณาอย่าตอบกลับ ' \
                       u'หากมีปัญหาใดๆเกี่ยวกับเว็บไซต์กรุณาติดต่อหน่วยข้อมูลและสารสนเทศ '
            message += u'\nThis email was sent by an automated system. Please do not reply.' \
                       u' If you have any problem about website, please contact the IT unit.'
            send_mail([u'{}@mahidol.ac.th'.format(status.account.staff.email)], title, message)
            flash(u'แก้ไขข้อมูลเรียบร้อย', 'success')
        return redirect(url_for('purchase_tracker.update_status', status_id=status.id, account_id=account_id))
    return render_template('purchase_tracker/edit_update_record.html',
                           account_id=account_id, form=form)

@purchase_tracker.route('/account/<int:account_id>/status/<int:status_id>/delete')
@finance_procurement_permission.require()
@login_required
def delete_update_status(account_id, status_id):
    if account_id:
        status = PurchaseTrackerStatus.query.get(status_id)
        flash(u'The update status has been removed.')
        db.session.delete(status)
        db.session.commit()
        return redirect(url_for('purchase_tracker.update_status', account_id=account_id))

@purchase_tracker.route('/create/<int:account_id>/activity', methods=['GET', 'POST'])
@finance_procurement_permission.require()
@login_required
def add_activity(account_id):
    activity = db.session.query(PurchaseTrackerActivity)
    form = CreateActivityForm()
    if request.method == 'POST':
        if form.validate_on_submit():
            new_activity = PurchaseTrackerActivity()
            form.populate_obj(new_activity)
            db.session.add(new_activity)
            db.session.commit()
            flash(u'บันทึกการเพิ่มกิจกรรมใหม่สำเร็จ.', 'success')
            return redirect(url_for('purchase_tracker.update_status', account_id=account_id))
        # Check Error
        else:
            for er in form.errors:
                flash(er, 'danger')
    return render_template('purchase_tracker/create_activity.html', form=form, activity=activity, account_id=account_id)

@purchase_tracker.route('/dashboard', methods=['GET', 'POST'])
def show_info_page():
    start_date = None
    end_date = None
    account_query = PurchaseTrackerAccount.query.all()
    form = ReportDateForm()
    if request.method == 'POST':
        if form.validate_on_submit():
            start_date = datetime.strptime(form.start_date.data, '%d-%m-%Y')
            end_date = datetime.strptime(form.end_date.data, '%d-%m-%Y')
            account_query = PurchaseTrackerAccount.query.filter(
                cast(PurchaseTrackerAccount.booking_date, Date) >= start_date) \
                .filter(cast(PurchaseTrackerAccount.booking_date, Date) <= end_date)
        else:
            flash(form.errors, 'danger')
    return render_template('purchase_tracker/info_page.html', account_query=account_query, form=form,
                           start_date=start_date, end_date=end_date)

@purchase_tracker.route('/dashboard/info/download', methods=['GET'])
def dashboard_info_download():
    records = []
    start_date = request.args.get('start_date')
    end_date = request.args.get('end_date')
    if start_date and end_date:
        accounts = PurchaseTrackerAccount.query.filter(cast(PurchaseTrackerAccount.booking_date, Date) >= start_date) \
            .filter(cast(PurchaseTrackerAccount.booking_date, Date) <= end_date)
    else:
        accounts = PurchaseTrackerAccount.query.all()

    for account in accounts:
        for record in account.records:
            records.append({
                u'เลขที่หนังสือ': u"{}".format(account.number),
                u'วันที่หนังสือ': u"{}".format(account.booking_date),
                u'ชื่อ': u"{}".format(account.subject),
                u'วงเงินหลักการ': u"{:,.2f}".format(account.amount),
                u'รูปแบบหลักการ': u"{}".format(account.formats),
                u'ผู้สร้าง account โดย': u"{}".format(account.staff.personal_info.fullname),
                u'หน่วยงาน/ภาควิชา': u"{}".format(account.staff.personal_info.org.name),
                u'กิจกรรม': u"{}".format(record.other_activity or record.activity.activity),
                u'ผู้รับผิดชอบ': u"{}".format(record.staff.personal_info.fullname),
                u'วันเริ่มกิจกรรม': u"{}".format(record.start_date),
                u'วันสิ้นสุดกิจกรรม': u"{}".format(record.end_date),
                u'หมายเหตุเพิ่มเติม': u"{}".format(record.comment),
                u'เวลาดำเนินกิจกรรม': u"{}".format(record.weekdays),
            })
    df = DataFrame(records)
    df.to_excel('account_summary.xlsx')
    return send_file(os.path.join(os.getcwd(), 'account_summary.xlsx'))

# @purchase_tracker.route('/personnel/personnel_index/e-form/create/<string:form_code>/<int:account_id>', methods=['GET', 'POST'])
# @login_required
# def create_form(account_id, form_code):
#     account = PurchaseTrackerAccount.query.get(account_id)
#     MTPCform = create_MTPCForm(acnt=account)
#     form = MTPCform()
#     if form.validate_on_submit():
#         new_form = PurchaseTrackerForm()
#         form.populate_obj(new_form)
#         new_form.staff = current_user
#         db.session.add(new_form)
#         db.session.commit()
#         flash(u'บันทึกข้อมูลสำเร็จ.', 'success')
#         form_letter(new_form, account)
#         return send_file('e-form.pdf')
#     # Check Error
#     else:
#         for er in form.errors:
#             flash("{}:{}".format(er,form.errors[er]), 'danger')
#     return render_template('purchase_tracker/personnel/create_form_{}.html'.format(form_code), form=form, account=account)
#
#
# sarabun_font = TTFont('Sarabun', 'app/static/fonts/THSarabunNew.ttf')
# pdfmetrics.registerFont(sarabun_font)
# style_sheet = getSampleStyleSheet()
# style_sheet.add(ParagraphStyle(name='ThaiStyle', fontName='Sarabun'))
# style_sheet.add(ParagraphStyle(name='ThaiStyleNumber', fontName='Sarabun', alignment=TA_RIGHT))
# style_sheet.add(ParagraphStyle(name='ThaiStyleCenter', fontName='Sarabun', alignment=TA_CENTER))
#
#
# def form_letter(form, account):
#     logo = Image('app/static/img/logo-MU.jpg', 60, 60)
#
#     def all_page_setup(canvas, doc):
#         canvas.saveState()
#         logo_image = ImageReader('app/static/img/logo-MU.jpg')
#         canvas.drawImage(logo_image, 10, 700, width=70, height=70)
#         canvas.restoreState()
#
#     doc = SimpleDocTemplate("app/e-form.pdf",
#                             pagesize=letter,
#                             rightMargin=72,
#                             leftMargin=72,
#                             topMargin=72,
#                             bottomMargin=18)
#
#
#     data = [ Paragraph(u'<font size=16>ภาควิชา / ศูนย์ {}</font>'.format(account.staff.personal_info.org.name), style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>ที่ {}</font>'.format(form.account.number), style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>วันที่ {}</font>'.format(form.account.creation_date), style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>เรื่อง {}</font>'.format(form.account.subject), style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>ข้าพเจ้า {}</font>'.format(form.name), style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>เหตุผลและความจำเป็นเร่งด่วนที่ต้องซื้อหรือจ้าง {}</font>'.format(form.reason),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>รายละเอียดของพัสดุที่ซื้อหรือจ้าง {}</font>'.format(form.account.desc),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>วงเงินที่ซื้อหรือจ้างในครั้งนี้เป็นเงินเท่าไหร่ {}</font>'.format(form.account.amount),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>จาก {}</font>'.format(form.account.amount),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>ตามใบส่งของ/ใบเสร็จรับเงินเล่มที่ {}</font>'.format(form.book),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>เลขที่ {}</font>'.format(form.number),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>วันที่ {}</font>'.format(form.receipt_date),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>โดยขอเบิกจ่ายจากเงิน {}</font>'.format(form.disbursement_method),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>ประจำปีงบประมาณ {}</font>'.format(form.financial_year),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>วันที่ {}</font>'.format(form.receipt_date),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>รหัสศูนย์ต้นทุน {}</font>'.format(form.cost_center),
#                        style=style_sheet['ThaiStyle']),
#              Paragraph(u'<font size=16>รหัสใบสั่งงานภายใน {}</font>'.format(form.internal_order),
#                        style=style_sheet['ThaiStyle']),
#              ]
#     data.append(Spacer(1, 12))
#
#     doc.build(data, onLaterPages=all_page_setup, onFirstPage=all_page_setup)


import numpy as np
from flask import Flask, request, jsonify, render_template
import pickle

# create the flask app
app = Flask(__name__)

# load the pickle model
model = pickle.load(open("model.pkl", "rb"))

# defining the homepage
@app.route("/")
def home():
    return render_template("index.html")

# defining the predict page
@app.route("/predict", methods=["POST"])
@app.route("/predict", methods=["POST"])
def predict():
    input = request.get_json()
    features = [np.array(list(input.values()))]
    prediction = model.predict(features)[0]
    print(features)
    if prediction == 0:
        prediction = "Drizzle"
    elif prediction == 1:
        prediction = "Foggy"
    elif prediction == 2:
        prediction = "Rain"
    elif prediction == 3:
        prediction = "Snow"
    elif prediction == 4:
        prediction = "Sunny"

    return jsonify({"prediction_text": "{}".format(prediction)})

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000)


#!/usr/bin/env python

"""
Calculate enrichment statistics for two sets of fasta files
Inputs:
   two fasta files to compare
   file containing patterns to check
Outputs:
   pickled dictionary of pattern enrichments
Ben Ober-Reynolds
"""

import os
import sys
import re
import time
import argparse
import numpy as np
import pandas as pd
import pickle
from Bio import SeqIO
from joblib import Parallel, delayed

def main():

    # set up command line argument parser
    parser = argparse.ArgumentParser(description='Calculate motif densities \
        for a target and a background set of fastas.')
    group = parser.add_argument_group('required arguments:')
    group.add_argument('-fi', '--fasta_of_interest', required=True,
        help='file containing clusters of interest')
    group.add_argument('-fb', '--background_fasta', required=True,
        help='file containing background clusters')
    group.add_argument('-pf', '--pattern_file', required=True,
        help='file containing patterns to check for. Format: \
        {pattern name}\\t{regex_pattern}')
    group = parser.add_argument_group('optional arguments')
    group.add_argument('-od', '--output_directory', default=".",
        help='output directory for statistics file and figures. \
        Default is current directory')
    group.add_argument('-op', '--output_prefix', default="enrichment",
        help='output prefix for results file and figures')
    group.add_argument('-isn', '--interesting_seq_name',
        default="Sequences of Interest",
        help='The name of the sequence of interest pool. Default is \
        "Sequences of Interest"')
    group.add_argument('-bsn', '--background_seq_name',
        default="Background Sequences", help='The name of the background \
        sequence pool. Default is "Background Sequences"')
    group.add_argument('-rc', '--reverse_comp', default="y",
        help='also calculate enrichment in reverse complement of each pool \
        [y/n]? Default is y.')
    group.add_argument('-nb', '--num_bootstraps', type=int, default=1000,
        help='number of times to resample pools for enrichment calculation. \
        Default is 1000.')
    group.add_argument('-n', '--num_cores', type=int, default=1,
        help='number of cores to use for bootstrapping.')

    # print help if no arguments provided
    if len(sys.argv) <= 1:
        parser.print_help()
        sys.exit()

    # parse command line arguments
    args = parser.parse_args()
    numCores = args.num_cores

    # Pre-defined variables, constants, and settings
    input_file_format = 'fasta'
    rev_c_tag = "Rev-Comp"
    output_prefix = time.strftime("%Y%m%d") + "_" + args.output_prefix
    pickle_file_ext = "p"

    # Do some error checking before running this long script:
    output_dir = args.output_directory
    if not os.path.isdir(output_dir):
        print("Error: invalid output directory. Exiting...")
        sys.exit()

    # Read in files:
    seqs_of_interest = read_fasta(args.fasta_of_interest, input_file_format)
    background_seqs = read_fasta(args.background_fasta, input_file_format)
    pattern_dict = read_pattern_file(args.pattern_file)

    # Find smallest pool size:
    pool_size = min([len(seqs_of_interest), len(background_seqs)])

    # seq pool dict:
    seq_pool_dict = {args.interesting_seq_name: seqs_of_interest,
    args.background_seq_name: background_seqs}

    # Results dictionary:
    density_result_dict = {}
    for pname in pattern_dict.keys():
        density_result_dict[pname] = {}

    # compare to reverse complement?
    if args.reverse_comp == 'y':
        interesting_seq_rc_name = args.interesting_seq_name + " " + rev_c_tag
        background_seq_rc_name = args.background_seq_name + " " + rev_c_tag
        rc_seqs_of_interest = reverse_comp(seqs_of_interest)
        rc_background_seqs = reverse_comp(background_seqs)
        seq_pool_dict[interesting_seq_rc_name] = rc_seqs_of_interest
        seq_pool_dict[background_seq_rc_name] = rc_background_seqs

    # calculate motif density for each pattern
    if numCores > 1:
        with Parallel(n_jobs=numCores, verbose=10) as parallel:
            for pname in pattern_dict.keys():
                for pool_name in seq_pool_dict.keys():
                    densities = []
                    print("Calculating density of pattern '{}' in pool '{}'\
                        ".format(pname, pool_name))
                    densities = parallel(delayed(calc_resampled_motif_density)\
                        (seq_pool_dict[pool_name], pool_size, pattern_dict[pname])
                    for i in range(args.num_bootstraps))
                    density_result_dict[pname][pool_name] = densities
    else:
        for pname in pattern_dict.keys():
            for pool_name in seq_pool_dict.keys():
                densities = []
                print("Calculating density of pattern '{}' in pool '{}'\
                    ".format(pname, pool_name))
                densities = [calc_resampled_motif_density(
                    seq_pool_dict[pool_name], pool_size, pattern_dict[pname])
                    for i in range(args.num_bootstraps)]
                density_result_dict[pname][pool_name] = densities

    # Dump results to pickle for latter replotting
    with open(output_dir + '/' + output_prefix + '.' + pickle_file_ext, 'wb') as f:
        pickle.dump(density_result_dict, f)

def read_fasta(filename, input_file_format):
    """
    Read in a fasta file, and return sequences as a list.
    Input: fasta filename
    Output: sequence array
    """
    fasta_list = []
    with open(filename, 'r') as f:
        for seq_rec in SeqIO.parse(f, input_file_format):
            seq_rec = seq_rec.upper()
            fasta_list.append(str(seq_rec.seq))
    return np.array(fasta_list)

def read_pattern_file(filename):
    """
    Read in a pattern file. Note that pattern files must be two-column,
    tab-delimited files with the first column being the pattern name, and
    the second column the regular expression defining that pattern.
    """
    pattern_dict = {}
    with open(filename, 'r') as f:
        for line in f:
            pname, reg_exp = line.strip().split('\t')
            reg_exp = re.compile(reg_exp)
            pattern_dict[pname] = reg_exp
    return pattern_dict

def reverse_comp(fasta_array):
    """
    Reverse complement a list of sequences
    Input: list of sequences
    Output: reverse complement of same sequence list
    """
    trans_table = str.maketrans('AGCT', 'TCGA')
    rev_list = []
    for seq in fasta_array:
        rev_list.append(seq.translate(trans_table)[::-1])
    return np.array(rev_list)

def calc_resampled_motif_density(seq_array, samp_size, regex):
    """
    Calculate the length-normalized density of a specific regular
    expression pattern in a resampled sequence pool.
    Inputs: list of sequences, number of seqs to draw, regular expression pattern
    Output: length-normalized motif density
    """
    resampled_pool = np.random.choice(seq_array, size=samp_size, replace=True)
    total_seq_space = 0
    patterns_found = 0
    for seq in resampled_pool:
        patterns_found += len(re.findall(regex, seq))
        total_seq_space += len(seq)
    return patterns_found/total_seq_space

if __name__ == '__main__':
    main()


import re
from collections import Counter

import gensim.parsing.preprocessing as proc
from gensim.utils import deaccent, to_unicode

####################
# Global variables
####################

# Load list of stopwords
with open('stopwords-es.txt', 'r') as file:
    stopwords = list()
    for line in file:
        stop_word = deaccent(line.strip('\n'))
        stopwords.append(stop_word)
stopwords = frozenset(stopwords)

# regexp for matching @usernames and #hashtags
tw_handles = r"([@][A-z]+)|([#][A-z]+)"
# for matching URLs
urls = r"((\w+:\/\/)[-a-zA-Z0-9:@;?&=\/%\+\.\*!'\(\),\$_\{\}\^~\[\]`#|]+)"
punctuation_es = r'([!¡"\#\$%\&\'\(\)\*\+,\-\./:;<=>\?\¿@\[\\\]\^_`\{\|\}\~])+'
# Master Regexp
multi_pattern = '|'.join([tw_handles, urls, punctuation_es])
non_plain_re = re.compile(multi_pattern, re.UNICODE)
####################

def remove_non_plain(document):
    """
    Replaces urls, @usernames, #tags, emojis and numbers
    with a ' ' (space). Also removes accents and punctuation
    to finally remove redundant whitespace and lowercase all
    characters
    :param document: string
    :return: processed unicode string
    """
    document = to_unicode(document)
    document = non_plain_re.sub(' ', document)
    document = proc.strip_non_alphanum(document)
    document = proc.strip_numeric(document)
    document = proc.strip_multiple_whitespaces(document)
    document = deaccent(document)
    return document.lower()

def process(document):
    """
    Tokenize a document (a tweet) removing:
    - punctuation
    - URLs and Twitter handles
    - Uppercases
    - Stopword
    - Whitespace
    :param document: string
    :returns: a list of strings
    """
    wordbag = list()
    for token in set(remove_non_plain(document).split()):
        if token not in stopwords and token != '' \
           and token != 'rt':
            wordbag.append(token)
    return wordbag

def init_counter(corpus):
    """
    Creates a collections.Counter object from a corpus.
    :param corpus: iterator or iterable of lists of strings
    :returns: Counter
    """
    ctr = Counter()
    for worbag in corpus:
        ctr.update(worbag)

    return ctr


from ocr import *
import glob
import matplotlib.pyplot as plt

weight_path = r'.\weights\ocr\yolov3-ocr_final.weights'
cfg_path = r'.\weights\ocr\yolov3-ocr.cfg'

def show_image(image):
    fig, ax1 = plt.subplots(1)
    ax1.imshow(image)
    plt.show()

image_paths = glob.glob("plates1/*.jpg")
image_paths = glob.glob("m/*.jpg")
image_paths = ['need contrast\\new\\534_0_plate.jpg']
id = 0
for img_path in image_paths:
    platereader = PlateReader()
    img, _, _, _ = platereader.load_image(img_path)
    #segmentedImage, plate_label = platereader.retrieve_label_and_segmentedImage_from(img_path, weight_path, cfg_path)
    segmentedImage, plate_label, sum_confidences = platereader.final_retrieve_label_and_segmentedImage_from_im(img, weight_path, cfg_path)
    #cv2.imshow(plate_label, segmentedImage)
    #show_image(segmentedImage)
    #cv2.imwrite(img_path, segmentedImage)
    cv2.imwrite(img_path[:-4]+str(id)+"_plate_"+plate_label+".jpg", segmentedImage)
    id += 1


#данный скрипт выдаёт словарь со всеми значениями и контекстами к ним (training set)

def scrape_txt():
    from Zvezda import define_microcontext
    from collections import Counter

    with open("static/file.txt") as f:
        text = f.read()
        word_meanings = {}

    stopwords_r = open('static/stopwords_r.txt')
    stopwords_r = stopwords_r.read()
    stopwords_r = stopwords_r.split(', ')

    for item in text.split('\n\n'):
        text, meaning = item.split('значение: ')
        found, context = define_microcontext(text) #возвращается найденная лемма "звезда" и контекст

        context = list(filter(lambda x: x not in stopwords_r, context))#убираем стоп-слова из контекста

        text = context

        context = Counter(context) #Частотный словарь

        context = {k: round(v / len(text), 3) for k, v in context.items()} #Присвоение MLE-коэффицента каждому слову контекста

        word_meanings[meaning] = context

    return word_meanings

#for k,v in scrape_txt().items():
#     print(k+':',v)


from scapy.all import IP, TCP, sniff
import logging    # first of all import the module

"""
Represents a network packet with source and destination IP addresses
"""
class Packet:
    def __init__(self,rules):
        self.rules = rules
    #True, "12.168.3.1009", "0", "10.0.0.100", "0"
    def get_summary(self, pkt):
        try:
            if TCP in pkt and IP in pkt:
                for rule in self.rules:
                    #print("src",pkt[IP].src, pkt[TCP].sport,"dst", pkt[IP].dst, pkt[TCP].dport)
                    #Check if the ip address and port number matches
                    if (pkt[IP].src == rule[1] and pkt[IP].dst == rule[3]):

                        if(True): #pkt[TCP].sport == rule[2]  ports changes a lot

                            #If they matches check if that rule is to be allowed or blocked
                            if rule[0] == True:
                                log = "Status: ", "BLOCK", "Src IP: ",pkt[IP].src, "Src Port:", pkt[TCP].sport,"Dst IP: ", pkt[IP].dst,"Dst Port:" ,pkt[TCP].dport
                                logging.basicConfig(filename='firewall.log', filemode='w', format="%(asctime)s %(message)s")
                                logging.warning(log)
                                print(log)

                            elif rule[0] == False:
                                log = "Status: ", "ALLOW", "Src IP: ",pkt[IP].src, "Src Port:", pkt[TCP].sport,"Dst IP: ", pkt[IP].dst,"Dst Port:" ,pkt[TCP].dport
                                logging.basicConfig(filename='firewall.log', filemode='w', format="%(asctime)s %(message)s")
                                logging.warning(log)
                                print(log)

                    else:
                        print("Src IP: ",pkt[IP].src, "Src Port:", pkt[TCP].sport,"Dst IP: ", pkt[IP].dst,"Dst Port:" ,pkt[TCP].dport)
            else:
                pass
                # return "Waiting for pkt...."
        except Exception as e:
            print(f"[ERR] Error could not get pkt: {e}")
            False

# capture = sniff(5)
# capture.summary()


"""initial

Revision ID: 2bbb456c6a28
Revises:
Create Date: 2023-11-04 14:46:55.786593

"""
import sqlalchemy as sa
from alembic import op  # pylint: disable=no-name-in-module

# revision identifiers, used by Alembic.
revision = '2bbb456c6a28'
down_revision = None
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        'user',
        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
        sa.Column('email', sa.String(length=64), nullable=False),
        sa.Column(
            'created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False
        ),
        sa.Column(
            'updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False
        ),
        sa.PrimaryKeyConstraint('id', name=op.f('pk_user')),
        sa.UniqueConstraint('id', name=op.f('uq_user_id')),
    )
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('user')
    # ### end Alembic commands ###


# _*_ coding:utf-8 _*_

from django.conf.urls import url

from .views import MessageDraftView, MessageInfoView, MessageManagementView, MessageSearchView,\
    MessageDraftFileUploadView, MessageCategoryManageView, ItemsMakeCountView, ItemReceiverCountView,\
    ItemsMakeSearchView,ItemReceiverSearchView

__author__ = 'haoSev7'
__date__ = '2017/5/26 21:59'

urlpatterns = [
    # 宣传管理信息申请表
    url(r'^messagedraft/$', MessageDraftView.as_view(), name='message_draft'),
    # 宣传信息统计页面
    url(r'^messageinfo/$', MessageInfoView.as_view(), name='message_info'),
    # 宣传信息管理页面
    url(r'^management/$', MessageManagementView.as_view(), name='management'),
    # 宣传信息查询页面
    url(r'^messagesearch/$', MessageSearchView.as_view(), name='messagesearch'),
    # 宣传信息申请表 附件上传
    url(r'^messagedraftupload/$', MessageDraftFileUploadView.as_view(), name='message_upload'),
    # 宣传信息类别管理页面
    url(r'^messagecategory/$', MessageCategoryManageView.as_view(), name='management_category'),
    # 宣传物资制作查询页面
    url(r'^itemmakesearch/$', ItemsMakeSearchView.as_view(), name='itemmakesearch'),
    # 宣传物资制作统计页面
    url(r'^itemmakecount/$', ItemsMakeCountView.as_view(), name='item_make_count'),
    # 宣传物资制作查询页面
    url(r'^itemreceiversearch/$', ItemReceiverSearchView.as_view(), name='itemreceiversearch'),
    # 宣传物资领用统计页面
    url(r'^itemreceivercount/$', ItemReceiverCountView.as_view(), name='item_receiver_count'),
]


# -*- coding: utf-8 -*-
"""
Created on Tue May  2 02:09:45 2023

@author: bat_j
"""

from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime, timedelta
from python_scripts.Raw_Data_Processing import Raw_Data_Processing
from python_scripts.Feature_engineering import Feature_engineering
from python_scripts.unit_test import moving_mean_unit_test, moving_median_unit_test
from python_scripts.Applying_ML import ML_model

default_args = {
    "owner": "airflow",
    "retries": 5,
    "retry_delay": timedelta(minutes=2),
    "start_date": datetime(2023,5,11)
    }

with DAG(
        default_args = default_args,
        dag_id = 'Data_Pipline_V1',
        description = 'DAG for the assessment',
        schedule_interval = '@daily'
    ) as dag:

    Data_Processing = PythonOperator(
        task_id = 'data_processing',
        python_callable = Raw_Data_Processing
        )

    Feature_Engineering = PythonOperator(
        task_id = 'feature_engineering',
        python_callable = Feature_engineering
        )

    Unit_test_1 = PythonOperator(
        task_id = 'unit_test_mean',
        python_callable = moving_mean_unit_test
        )

    Unit_test_2 = PythonOperator(
        task_id = 'unit_test_median',
        python_callable = moving_median_unit_test
        )

    Machine_learning_model = PythonOperator(
        task_id = 'Machine_learning_model',
        python_callable = ML_model
        )

    Data_Processing >> Feature_Engineering
    Feature_Engineering >> Unit_test_1
    Feature_Engineering >> Unit_test_2
    Unit_test_1 >> Machine_learning_model
    Unit_test_2 >> Machine_learning_model


from collections import deque
from copy import deepcopy

def solution(name):
    answer = 0
    queue = deque()
    visited = [[False] for _ in name]
    dd = [1, -1]
    init = 'A' * len(name)

    # position, movements
    queue.append([0, upAndDown(name[0]), name[0] + init[1:]])
    visited[0] = True
    result = 987654321

    while queue:
        p, m, copied_name = queue.popleft()

        if copied_name == name:
            result = min(result, m)
            break

        for i in range(2):
            _p = p + dd[i]

            if _p == -1: _p = len(name) - 1
            elif _p == len(name):
                _p = 0

            if not visited[_p]: continue
            visited[_p] = True
            if _p == len(name) - 1:
                queue.append([_p, m + upAndDown(name[_p]) + 1, copied_name[0:_p] + name[_p]])
            else:
                queue.append([_p, m + upAndDown(name[_p]) + 1, copied_name[0:_p] + name[_p] + copied_name[_p + 1:]])

    answer = result
    return answer

def upAndDown(alpha):
    # starts with 'A'
    # go up
    up = ord('Z') - ord(alpha) + 1

    # go down
    down = ord(alpha) - ord('A')

    return min(up, down)

print(solution("JEROEN"))


from wordcloud import WordCloud, STOPWORDS
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

file_path = "Classic Computer Science Problems in Python by David Kopec (z-lib.org).txt"
file_handle = open(file_path).read()
stopwords = set(STOPWORDS)
stopwords.add("will")

book = WordCloud(max_words=5000, stopwords=stopwords)
book.generate(file_handle)
plt.imshow(book, interpolation='bilinear')
plt.axis("off")
plt.show()

image_file = "griffin-silhouette-ancient-mythology-fantasy-vector-18016755.jpg"

# create mask
a_mask = np.array(Image.open(image_file))

wc = WordCloud(background_color="black", max_words=2000, mask=a_mask,
               stopwords=stopwords)
wc.generate(file_handle)

plt.figure(figsize=(8,6), dpi=120)
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()