import hashlib
import time

from django.core.cache import caches, DEFAULT_CACHE_ALIAS

from corehq.project_limits.rate_counter.interfaces import AbstractRateCounter

REDIS = caches[DEFAULT_CACHE_ALIAS]
LOCMEM = caches['locmem']

class SlidingWindowRateCounter(AbstractRateCounter):
    """
    A "Sliding Window Over Fixed Grains" approach that approximates perfect sliding window

    A perfect sliding window approach would require keeping the timestamp of every event
    and count the number that fall between now and now - window_duration;
    for lower memory and performance overhead we instead approximate that
    by dividing time up into fixed "grains", i.e. sub-windows,
    over which we slide the larger window.
    For the earliest grain which is sliding out of the window,
    we assume linear distribution of events over time, and thus compute its contribution
    to the total as (% overlap of grain with our window) * (events in grain).

    See a description of this approach (with grains_per_window=1) here:
    https://konghq.com/blog/how-to-design-a-scalable-rate-limiting-algorithm/

    """
    def __init__(self, key, window_duration, window_offset=0, grains_per_window=1,
                 memoize_timeout=15.0, _FixedWindowRateCounter=None):
        """

        :param key: short description of the window e.g. "week"
        :param window_duration: length of the window in seconds
        :param window_offset: offset from epoch of window boundary
        :param grains_per_window: How many grains the window should be divided into
            (higher number = more accuracy)
        :param memoize_timeout: how long to memoize the information in memory in seconds
            This is the upper limit on how long a `get` could return a stale value.
        """
        _FixedWindowRateCounter = _FixedWindowRateCounter or FixedWindowRateCounter
        self.key = key
        self.window_duration = window_duration
        self.window_offset = window_offset
        self.grains_per_window = grains_per_window
        self.memoize_timeout = memoize_timeout
        self.grain_duration = window_duration / grains_per_window
        self.grain_counter = _FixedWindowRateCounter(
            key=key,
            window_duration=self.grain_duration,
            window_offset=window_offset,
            keep_windows=grains_per_window + 1,
            memoize_timeout=memoize_timeout
        )

    def get(self, scope, timestamp=None):
        if timestamp is None:
            timestamp = time.time()
        counts = [
            self.grain_counter.get(scope, timestamp - i * self.grain_duration,
                                   key_is_active=(i == 0))
            for i in range(self.grains_per_window + 1)
        ]
        earliest_grain_count = counts.pop()
        # This is the percentage of the way through the current grain we are
        progress_in_current_grain = (timestamp % self.grain_duration) / self.grain_duration
        # This is the count from the percentage of the earliest grain that should count
        contribution_from_earliest = earliest_grain_count * (1 - progress_in_current_grain)
        return sum(counts) + contribution_from_earliest

    def retry_after(self):
        """Calculates the time (in seconds) left in the current grain"""
        timestamp = time.time()
        progress_in_current_grain = (timestamp % self.grain_duration) / self.grain_duration
        progress_left_in_grain = 1 - progress_in_current_grain
        return progress_left_in_grain * self.grain_duration

    def increment(self, scope, delta=1, timestamp=None):
        # this intentionally doesn't return because this is the active grain count,
        # not the total that would be returned by get
        self.grain_counter.increment(scope, delta, timestamp=timestamp)

    def increment_and_get(self, scope, delta=1, timestamp=None):
        self.increment(scope, delta, timestamp=timestamp)
        return self.get(scope, timestamp=timestamp)

class FixedWindowRateCounter(AbstractRateCounter):
    def __init__(self, key, window_duration, window_offset=0, keep_windows=1,
                 memoize_timeout=15.0, _CounterCache=None):
        """
        :param key: short description of the window e.g. "week"
        :param window_duration: length of the window in seconds
        :param window_offset: offset of window boundary in seconds from the epoch
        :param keep_windows: number of windows to retain (including current one)
        :param memoize_timeout: how long to memoize the information in memory in seconds
            This is the upper limit on how long a `get` could return a stale value.
        """
        _CounterCache = _CounterCache or CounterCache
        assert keep_windows >= 1
        self.key = key
        self.window_duration = window_duration
        self.window_offset = window_offset
        self.counter = _CounterCache(memoize_timeout, timeout=keep_windows * window_duration)

    @staticmethod
    def _digest(string):
        return hashlib.sha1(string.encode('utf-8')).hexdigest()

    def _cache_key(self, scope, timestamp=None):
        if timestamp is None:
            timestamp = time.time()
        if isinstance(scope, str):
            scope = (scope,)
        return self._digest('fwrc-{}-{}-{}'.format(
            self.key,
            ':'.join(map(self._digest, scope)),
            int((timestamp - self.window_offset) / self.window_duration)
        ))

    def get(self, scope, timestamp=None, key_is_active=True):
        return self.counter.get(self._cache_key(scope, timestamp=timestamp), key_is_active=key_is_active)

    def increment_and_get(self, scope, delta=1, timestamp=None):
        return self.counter.incr(self._cache_key(scope, timestamp=timestamp), delta)

    def increment(self, scope, delta=1, timestamp=None):
        self.increment_and_get(scope, delta, timestamp=timestamp)

class CounterCache(object):
    def __init__(self, memoized_timeout, timeout, local_cache=LOCMEM, shared_cache=REDIS):
        self.memoized_timeout = memoized_timeout
        self.timeout = int(timeout)
        self.local_cache = local_cache
        self.shared_cache = shared_cache

    def incr(self, key, delta=1):
        value = self.shared_cache.incr(key, delta, ignore_key_check=True)
        if value == 1:
            self.shared_cache.expire(key, timeout=self.timeout)
        self.local_cache.set(key, value, timeout=self.memoized_timeout)
        return value

    def get(self, key, key_is_active=True):
        """
        :param key: Cache key
        :param key_is_active: Whether you believe the key is being actively updated
            If not, then use the longer timeout for local memory cache as well.
        """
        local_timeout = self.memoized_timeout if key_is_active else self.timeout
        value = self.local_cache.get(key, default=None)
        if value is None:
            value = self.shared_cache.get(key, default=0)
            self.local_cache.set(key, value, timeout=local_timeout)
        assert value is not None
        return value


# chat/urls.py
from django.urls import path, include
from rest_framework import routers
from . import views

router = routers.DefaultRouter()
router.register(r'sessions', views.EditSessionView, 'Sessions')
router.register(r'collaborate', views.EditSessionAllowedView, 'ListSessions')

urlpatterns = [
    path('', include(router.urls)),
    path('', views.index, name='index'),
    path('<str:room_name>/', views.room, name='room'),
    path('account/register', views.UserCreate.as_view())
]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  4 13:35:18 2021

@author: mike_ubuntu
"""

from typing import Union, Callable
import numpy as np
from sklearn.linear_model import Lasso

import epde.globals as global_var
from epde.operators.utils.template import CompoundOperator
from epde.structure.main_structures import Equation

class LASSOSparsity(CompoundOperator):
    """
    The operator, which applies LASSO regression to the equation object to detect the
    valuable term coefficients.

    Notable attributes:
    -------------------

    params : dict
        Inhereted from the ``CompoundOperator`` class.
        Parameters of the operator; main parameters:

            sparsity - value of the sparsity constant in the LASSO operator;

    g_fun : np.ndarray or None:
        values of the function, used during the weak derivatives estimations.

    Methods:
    -----------
    apply(equation)
        calculate the coefficients of the equation, that will be stored in the equation.weights np.ndarray.

    """
    key = 'LASSOBasedSparsity'

    def apply(self, objective : Equation, arguments : dict):
        """
        Apply the operator, to fit the LASSO regression to the equation object to detect the
        valueable terms. In the Equation class, a term is selected to represent the right part of
        the equation, and its values are used here as the target, and the values of the other
        terms are utilizd as the features. The method does not return the vector of coefficients,
        but rather assigns the result to the equation attribute ``equation.weights_internal``

        Parameters:
        ------------
        equation : Equation object
            the equation object, to that the coefficients are obtained.

        Returns:
        ------------
        None
        """
        # print(f'Metaparameter: {objective.metaparameters}, objective.metaparameters[("sparsity", objective.main_var_to_explain)]')
        self_args, subop_args = self.parse_suboperator_args(arguments = arguments)

        estimator = Lasso(alpha = objective.metaparameters[('sparsity', objective.main_var_to_explain)]['value'],
                          copy_X=True, fit_intercept=True, max_iter=1000,
                          positive=False, precompute=False, random_state=None,
                          selection='cyclic', tol=0.0001, warm_start=False)
        _, target, features = objective.evaluate(normalize = True, return_val = False)
        self.g_fun_vals = global_var.grid_cache.g_func.reshape(-1)

        estimator.fit(features, target, sample_weight = self.g_fun_vals)
        objective.weights_internal = estimator.coef_

    def use_default_tags(self):
        self._tags = {'sparsity', 'gene level', 'no suboperators', 'inplace'}


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden
from django.views.decorators.csrf import csrf_exempt
from django.conf import settings

from linebot import LineBotApi, WebhookParser
from linebot.exceptions import InvalidSignatureError, LineBotApiError
from linebot.models import MessageEvent, TextSendMessage, JoinEvent, FollowEvent, MemberJoinedEvent, TextSendMessage, TemplateSendMessage, ButtonsTemplate, URITemplateAction

from leetcodelinebot.models import ReportLog, Users
from leetcodelinebot.lineNotify import line_notify_send_message
from leetcodelinebot.myself import myself
from datetime import datetime
from pytz import timezone
import re

line_bot_api = LineBotApi(settings.LINE_CHANNEL_ACCESS_TOKEN)
parser = WebhookParser(settings.LINE_CHANNEL_SECRET)

@csrf_exempt
def callback(request):
    if request.method == 'POST':
        signature = request.META['HTTP_X_LINE_SIGNATURE']
        body = request.body.decode('utf-8')

        try:
            events = parser.parse(body, signature)
        except InvalidSignatureError:
            return HttpResponseForbidden()
        except LineBotApiError:
            return HttpResponseBadRequest()

        for event in events:
            if isinstance(event, JoinEvent):  # 如果有加入聊天室的事件
                line_bot_api.reply_message(  # 回復傳入的訊息文字
                    event.reply_token,
                    TemplateSendMessage(
                        alt_text='哈囉 哈囉',
                        template=ButtonsTemplate(
                            thumbnail_image_url='https://picx.zhimg.com/v2-e1425095196ac03e4c781a42be0cdc26_r.jpg',  # 替換成你要顯示的圖片網址
                            title='大家好我是Tasktrackbot',
                            text='請先將我加入好友才可以為你服務',
                            actions=[
                                URITemplateAction(
                                    label='加入好友',
                                    uri='https://liff.line.me/1645278921-kWRPP32q/?accountId=615veimk'
                                )
                            ]
                        )
                    )
                )

            elif isinstance(event, FollowEvent):  # 如果是加好友事件
                user_id = event.source.user_id
                profile = line_bot_api.get_profile(user_id)

                user = Users.objects(user_id=user_id).first()
                if user:
                    user.display_name = profile.display_name
                    user.status_message = profile.status_message
                    user.picture_url = profile.picture_url
                    user.save()
                else:
                    taiwan_tz = timezone('Asia/Taipei')
                    taiwan_time = datetime.now(taiwan_tz)
                    users = Users(
                        user_id = user_id,
                        name = profile.display_name,
                        status_message = profile.status_message,
                        picture_url = profile.picture_url,
                        punish = 0,
                        created_at=taiwan_time
                    )
                    users.save()

                line_bot_api.reply_message(
                    event.reply_token,
                    TextSendMessage(text='哈囉~我是Tasktrackbot\n很高興能為你服務')  # 回覆未加入好友訊息
                )

            elif isinstance(event, MemberJoinedEvent):  # 如果是新的使用者加入群組事件
                line_bot_api.reply_message(  # 回復傳入的訊息文字
                    event.reply_token,
                    TemplateSendMessage(
                        alt_text='歡迎 歡迎',
                        template=ButtonsTemplate(
                            thumbnail_image_url='https://picx.zhimg.com/v2-e1425095196ac03e4c781a42be0cdc26_r.jpg',  # 替換成你要顯示的圖片網址
                            title='歡迎新朋友~我是Tasktrackbot',
                            text='請先將我加入好友才可以為你服務',
                            actions=[
                                URITemplateAction(
                                    label='加入好友',
                                    uri='https://liff.line.me/1645278921-kWRPP32q/?accountId=615veimk'
                                )
                            ]
                        )
                    )
                )
            elif isinstance(event, MessageEvent):  # 如果有訊息事件
                try:
                    user_id = event.source.user_id
                    profile = line_bot_api.get_profile(user_id)
                    user = Users.objects(user_id=user_id).first()
                    if user:
                        user.display_name = profile.display_name
                        user.status_message = profile.status_message
                        user.picture_url = profile.picture_url
                        user.save()
                    else:
                        taiwan_tz = timezone('Asia/Taipei')
                        taiwan_time = datetime.now(taiwan_tz)
                        users = Users(
                            user_id = user_id,
                            name = profile.display_name,
                            status_message = profile.status_message,
                            picture_url = profile.picture_url,
                            punish = 0,
                            created_at=taiwan_time
                        )
                        users.save()
                    if event.message.text == '查詢紀錄':
                        reply_text = get_report_stats(user_id)  # 呼叫函式取得 ReportLog 統計數據
                        line_bot_api.reply_message(
                            event.reply_token,
                            TextSendMessage(text=profile.display_name + reply_text)  # 構建回覆訊息
                        )
                    elif event.message.text == '回報進度':
                        line_bot_api.reply_message(
                            event.reply_token,
                            TextSendMessage(text="輸入格式：「完成 1」")  # 構建回覆訊息
                        )
                    elif event.message.text is not None and event.message.text.startswith('完成'):
                        topic = extract_topic_from_message(event.message.text)
                        if topic is not None:
                            reply_text = write_to_report_log(user_id=user_id, name=profile.display_name, topic=topic, done=True)
                            line_bot_api.reply_message(
                                event.reply_token,
                                TextSendMessage(text=reply_text)  # 回覆新增成功訊息
                            )
                        else:
                            line_bot_api.reply_message(
                                event.reply_token,
                                TextSendMessage(text='未提取到數字，舉例:[完成 1]')  # 回覆未提取到數字訊息
                            )
                    elif event.message.text is not None and event.message.text.startswith('：'):
                        isMe = myself(user_id)
                        if isMe:
                            line_notify_send_message(event.message.text.replace('：', ''))
                except LineBotApiError:
                    line_bot_api.reply_message(
                        event.reply_token,
                        TextSendMessage(text='加我好友再說話！')  # 回覆未加入好友訊息
                    )
        return HttpResponse()
    else:
        return HttpResponseBadRequest()

def write_to_report_log(user_id, name, topic, done):
    # 检查是否已存在相同用户和相同topic的记录
    existing_log = ReportLog.objects(user_id=user_id, topic=topic).first()

    if existing_log:
        reply_text = f"第{topic}題您已經完成了，請更換題目"
        return reply_text

    taiwan_tz = timezone('Asia/Taipei')
    taiwan_time = datetime.now(taiwan_tz)
    formatted_time = taiwan_time.strftime('%Y/%m/%d %H:%M')

    report_log = ReportLog(
        user_id=user_id,
        name=name,
        topic=topic,
        done=done,
        created_at=taiwan_time
    )

    report_log.save()

    user = Users.objects(user_id=user_id).first()

    user.punish +=1
    user.save()

    reply_text = f"成功新增紀錄 {formatted_time}"
    return reply_text

def get_report_stats(user_id):
    taiwan_tz = timezone('Asia/Taipei')
    now = datetime.now(taiwan_tz)
    today = now.date()
    start_of_day = datetime.combine(today, datetime.min.time())
    end_of_day = datetime.combine(today, datetime.max.time())

    all_topics = ReportLog.objects(user_id=user_id).distinct("topic")
    total_count = len(all_topics)

    today_topics = ReportLog.objects(user_id=user_id, created_at__gte=start_of_day, created_at__lte=end_of_day).distinct("topic")
    today_count = len(today_topics)

    reply_text = f"過去總共完成了{total_count}題測驗，今日已完成{today_count}題"

    return reply_text

def extract_topic_from_message(message):
    # 刪除所有空格
    message = message.replace(" ", "")
    # 使用正則表達式提取數字部分
    match = re.search(r'完成(\d+)|(\d+)完成', message)
    if match:
        # 提取到數字部分，回傳作為 topic
        topic = match.group(1) or match.group(2)
        return topic
    # 若未提取到數字部分，回傳 None
    return None


import pygame
import math

from physx import PhysX_Object

class Planet(PhysX_Object):

	def __init__(self, location, category = 2):
		PhysX_Object.__init__(self, location, (0, 0), (0, 0), (200, 200), 1e10, 1)

		source = "../img/planet_" + str(category) + ".png"
		self.load_image(pygame.image.load(source))

	def update(self, time_elapsed, events):
		PhysX_Object.update(self, time_elapsed, events)

	def gravitate(self, other):
		# range-scaling grav
		'''d_x = math.fabs(self.x_loc) - math.fabs(other.x_loc)
		d_y = math.fabs(self.y_loc) - math.fabs(other.y_loc)
		r = ((d_x ** 2) + (d_y ** 2)) ** 0.5

		G = 6.67e-5

		acc = G * self.mass / (r ** 1)

		other.x_acc += acc * (d_x / r)
		other.y_acc += acc * (d_y / r)'''

		# constant range-restricted grav
		d_x = math.fabs(self.x_loc) - math.fabs(other.x_loc)
		d_y = math.fabs(self.y_loc) - math.fabs(other.y_loc)
		r = ((d_x ** 2) + (d_y ** 2)) ** 0.5

		G = 6.67e-9
		f_g = G * self.mass

		if (r < 200):
			f_x = (d_x / r) * f_g
			f_y = (d_y / r) * f_g

			other.x_acc += f_x
			other.y_acc += f_y


import pandas as pd
import pickle
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.model_selection import train_test_split

def fit(extractor: str):

    '''
    This function trains and save the stroke classification model using features previously extracted from the MGABT {pixel, percent} extractors.

    :param extractor: Type of extractor used: *percent* referring to MGABTD_percent and *pixel* referring to MGABTD_pixel
    :type extractor: str
    '''

    df = pd.read_csv(f'train/dataset_{extractor}.csv', header=0)

    X_train, _, y_train, _ = train_test_split(
            df.drop(['target'], axis=1),
            df['target'],
            train_size=0.7,
            stratify=df['target']
    )
    clf = ExtraTreesClassifier()
    clf.fit(X_train, y_train)

    with open(f'model/extratree_{extractor}.pkl', 'wb') as outp:
        pickle.dump(clf, outp)

    return clf


"""
Register error handler in flask framework

"""
from flask import jsonify, Blueprint
from exception import RequestDataFormatException
from exception import CaptchaValidException
from exception import GetCaptchaException

errorHandler = Blueprint("errorHandler", __name__)

@errorHandler.app_errorhandler(RequestDataFormatException)
def handleRequestDataFormatException(error):
    response = jsonify(error.toDict())
    response.statusCode = error.statusCode
    return response

@errorHandler.app_errorhandler(CaptchaValidException)
def handleCaptchaValidException(error):
    response = jsonify(error.toDict())
    response.statusCode = error.statusCode
    return response

@errorHandler.app_errorhandler(GetCaptchaException)
def handleGetCaptchaException(error):
    response = jsonify(error.toDict())
    response.statusCode = error.statusCode
    return response


from sklearn.base import is_classifier, is_regressor

from ._estimator import classifier, regressor

__all__ = ['validate_config', 'validate_estimator']

def validate_config(config):
    if config is None or len(config) == 0:
        raise ValueError(
            "Invalid 'config', 'config' should be a list"
            " of (estimator, params) tuples."
        )

    estimators, params = zip(*config)
    return estimators, params

def validate_estimator(estimator, estimator_type=None):
    assert estimator_type in {'classifier', 'regressor', None}
    _estimator = classifier if estimator_type == 'classifier' else regressor

    if estimator in _estimator:
        return _estimator[estimator]()

    elif isinstance(estimator, type):
        return estimator()

    elif is_classifier(estimator) or is_regressor(estimator):
        return estimator

    raise ValueError


import json

from django.core.serializers.json import DjangoJSONEncoder
from django.forms import Media, widgets
from django.forms.renderers import get_default_renderer
from django.utils.encoding import force_str
from django.utils.functional import Promise, cached_property
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

from .config import CONFIG_TOOLS, PLUGINS, PLUGINS_KEYS, VERSION

class LazyEncoder(DjangoJSONEncoder):
    def default(self, obj):
        if isinstance(obj, Promise):
            return force_str(obj)
        return super().default(obj)

json_encode = LazyEncoder().encode

class EditorJsWidget(widgets.Textarea):
    def __init__(self, plugins=None, tools=None, config=None, **kwargs):
        self.plugins = plugins
        self.tools = tools
        self.config = config

        # Fix "__init__() got an unexpected keyword argument 'widget'"
        widget = kwargs.pop('widget', None)
        if widget:
            self.plugins = widget.plugins
            self.tools = widget.tools
            self.config = widget.config

        super().__init__(**kwargs)

    def configuration(self):
        tools = {}
        config = self.config or {}

        if self.plugins or self.tools:
            custom_tools = self.tools or {}
            # get name packages without version
            plugins = ['@'.join(p.split('@')[:2])
                       for p in self.plugins or PLUGINS]

            for plugin in plugins:
                plugin_key = PLUGINS_KEYS.get(plugin)

                if not plugin_key:
                    continue

                plugin_tools = custom_tools.get(
                    plugin_key) or CONFIG_TOOLS.get(plugin_key) or {}
                plugin_class = plugin_tools.get('class')

                if plugin_class:

                    tools[plugin_key] = custom_tools.get(
                        plugin_key, CONFIG_TOOLS.get(plugin_key)
                    )

                    tools[plugin_key]['class'] = plugin_class

                    custom_tools.pop(plugin_key, None)

            if custom_tools:
                tools.update(custom_tools)
        else:  # default
            tools.update(CONFIG_TOOLS)

        config.update(tools=tools)
        return config

    @cached_property
    def media(self):
        js_list = [
            '//cdn.jsdelivr.net/npm/@editorjs/editorjs@' + VERSION  # lib
        ]

        plugins = self.plugins or PLUGINS

        if plugins:
            js_list += ['//cdn.jsdelivr.net/npm/' + p for p in plugins]

        js_list.append('django-editorjs-fields/js/django-editorjs-fields.js')

        return Media(
            js=js_list,
            css={
                'all': [
                    'django-editorjs-fields/css/django-editorjs-fields.css'
                ]
            },
        )

    def render(self, name, value, attrs=None, renderer=None):
        if value is None:
            value = ''

        if renderer is None:
            renderer = get_default_renderer()

        return mark_safe(renderer.render("django-editorjs-fields/widget.html", {
            'widget': {
                'name': name,
                'value': conditional_escape(force_str(value)),
                'attrs': self.build_attrs(self.attrs, attrs),
                'config': json_encode(self.configuration()),
            }
        }))


import csv
from pathlib import Path

import numpy as np

from src.experiments.vlc_simulator_validation.analysis.tek_reader.tek_result import TekResult

class TekReader:
    @staticmethod
    def read(csv_path: Path) -> TekResult:
        # Initialize TekCsv instance
        tek_result = TekResult()

        # Initialize lists to store data
        time_values = []
        signal_values = []
        tek_result.metadata = {}

        try:
            with open(csv_path, 'r') as csv_file:
                # Create a CSV reader
                csv_reader = csv.reader(csv_file)

                for row in csv_reader:
                    if len(row) >= 5:  # Ensure there are at least 5 columns in each row
                        # Extract data from columns 4 and 5 (0-based indexing)
                        time_value = float(row[3])
                        signal_value = float(row[4])

                        time_values.append(time_value)
                        signal_values.append(signal_value)

                    if len(row) >= 2:  # Ensure there are at least 2 columns in each row
                        # Extract data from columns 1 and 2 and build a dictionary
                        column1_value = row[0]
                        column2_value = row[1]

                        tek_result.metadata[column1_value] = column2_value

            # Convert the lists to NumPy arrays
            tek_result.time = np.array(time_values)
            tek_result.signal = np.array(signal_values)

            return tek_result

        except Exception as e:
            raise e


import scrapy
from pathlib import Path

import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from helpers import leagues, seasons

class StadiumsSpider(scrapy.Spider):
    name = "stadiums"

    def start_requests(self):
        urls = []
        for league in leagues:
            for season in seasons:
                url = f"https://www.transfermarkt.com/laliga/besucherzahlen/wettbewerb/{league}/plus/1?saison_id={season}"
                urls.append(url)

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        season = response.url[-4:]
        league = response.url.split("/")[-3]

        base_path = Path(__file__).parent
        file_path = (base_path / f"../stadium-pages/stadiums-{league}-{season}.html").resolve()

        with open(file_path, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file at {file_path}')


from pathlib import Path

import pytest
from airflow.models import DagBag, Variable

DAGS_FOLDER = f"{Path(__file__).resolve().parent.parent}/dags"

@pytest.fixture(autouse=True)
def env_vars_setup(monkeypatch):
    mock_env_vars = {
        "AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT": "google-cloud-platform://",
        "DAGS_FOLDER": DAGS_FOLDER,
    }
    for k, v in mock_env_vars.items():
        monkeypatch.setenv(k, v)

@pytest.fixture(autouse=True)
def airflow_vars_setup(monkeypatch):
    mock_airflow_vars = {
        "ethereum_destination_dataset_project_id": "test",
        "ethereum_load_all_partitions": False,
        "ethereum_output_bucket": "test",
        "ethereum_parse_destination_dataset_project_id": "test",
        "ethereum_provider_uris": "test",
    }
    monkeypatch.setattr(Variable, "get", mock_airflow_vars.get)

@pytest.fixture
def dag_bag():
    yield DagBag(dag_folder=DAGS_FOLDER, include_examples=False)

def test_no_import_errors(dag_bag):
    assert len(dag_bag.import_errors) == 0, "No Import Failures"


from PreProcess import getProcessedConcepts
from nltk.corpus import wordnet as wn
import time

current_milli_time = lambda: int(round(time.time() * 1000))

file = open("C:\\Users\\myste\\Google Drive\\CMU\\Sem1\\Research\\Concepts\\concepts.txt", "r", encoding='utf8')
lines = []

lines = concepts = getProcessedConcepts()

# Filling up synsets for each concept
synsets = []
for line in lines:
    sublist = []
    for word in line.split(" "):
        for synset in wn.synsets(word):
            sublist.append(synset)
            break;
    synsets.append(sublist)
print(synsets)

def wupBestMatch(query):
    query = query.lower()
    print("Finding best match for " + query)
    st = current_milli_time()
    querySyns = []
    for queryWord in query.split(" "):
        for querySyn in wn.synsets(queryWord):
            querySyns.append(querySyn)
            break;

    # Finding similar concept
    maxScoreWup = 0
    maxIdxWup = 0
    idx = 0
    for synList in synsets:
        scoreWup = 0
        noMatchWup = 0
        for querySyn in querySyns:
            for syn in synList:
                indScore = wn.wup_similarity(querySyn, syn);
                if indScore is not None:
                    scoreWup += indScore
                else:
                    noMatchWup += 1
        if len(querySyns) * len(synList) - noMatchWup > 0:
            scoreWup /= len(querySyns) * len(synList) - noMatchWup
            if maxScoreWup < scoreWup:
                maxScoreWup = scoreWup
                maxIdxWup = idx
        idx += 1
    print("Best match Wup similarity: " + lines[maxIdxWup] + " :: ", maxScoreWup)
    print("Found in %d millisecs" % (current_milli_time() - st))

def lchBestMatch(query):
    query = query.lower()
    print("Finding best match for " + query)
    st = current_milli_time()
    querySyns = []
    for queryWord in query.split(" "):
        for querySyn in wn.synsets(queryWord):
            querySyns.append(querySyn)
            break;

    # Finding similar concept
    maxScoreLch = 0
    maxIdxLch = 0
    idx = 0
    for synList in synsets:
        scoreLch = 0
        noMatchLch = 0
        for querySyn in querySyns:
            for syn in synList:
                indScore = 0
                if querySyn._pos == syn._pos:
                    indScore = wn.lch_similarity(querySyn, syn)
                if (indScore is not None):
                    scoreLch += indScore
                else:
                    noMatchLch += 1
        if len(querySyns)*len(synList) - noMatchLch > 0:
            scoreLch /= len(querySyns)*len(synList) - noMatchLch
            if maxScoreLch < scoreLch:
                maxScoreLch = scoreLch
                maxIdxLch = idx
        idx += 1
    print("Best match Lch similarity: " + lines[maxIdxLch] + " :: ",maxScoreLch)
    print("Found in %d millisecs" % (current_milli_time() - st))

# Testing
query1 = "hgfdgfkhuflkdflu"
query2 = "No white trains"
query3 = "Regular soccer match"
query4 = "chicken breasts"
query5 = "buildings and trees"
query6 = "bikes"
wupBestMatch(query1)
lchBestMatch(query1)
print('----------------------------')
wupBestMatch(query2)
lchBestMatch(query2)
print('----------------------------')
wupBestMatch(query3)
lchBestMatch(query3)
print('----------------------------')
wupBestMatch(query4)
lchBestMatch(query4)
print('----------------------------')
wupBestMatch(query5)
lchBestMatch(query5)
print('----------------------------')
wupBestMatch(query6)
lchBestMatch(query6)


import argparse
from collections import OrderedDict
from typing import Dict, List, Optional, Tuple

import numpy as np
import torch
from pytorch_lightning import LightningModule, Trainer, seed_everything
from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning.plugins import DataParallelPlugin, DDP2Plugin
from torch import Tensor, optim
from torch.optim.optimizer import Optimizer
from torch.utils.data import DataLoader

from zenquant.agents.common.experience_source import Experience, ExperienceSourceDataset
from zenquant.agents.common.loss import dqn_loss
from zenquant.agents.common.agent import ValueAgent
from zenquant.agents.common.memory import MultiStepBuffer
from zenquant.agents.common.networks import CNN
from vnpy.trader.constant import (
    Status,
    Direction,
    Offset,
    Exchange
)

from zenquant.env.backtest_env import BacktestEnv

class DQN(LightningModule):
    def __init__(
        self,
        env: 'BacktestEnv',
        test_env: Optional['BacktestEnv'] = None,
        **kwargs,
    ):
        self.eps_start = kwargs.get("eps_start",1.0)
        self.eps_end = kwargs.get("eps_end",0.02)
        self.eps_last_frame = kwargs.get("eps_last_frame",150000)
        self.sync_rate = kwargs.get("sync_rate",1000)
        self.gamma = kwargs.get("gamma",0.99)
        self.learning_rate = kwargs.get("learning_rate",0.001)
        self.batch_size = kwargs.get("batch_size",32)
        self.replay_size = kwargs.get("replay_size",10000)
        self.warm_start_size =  kwargs.get("warm_start_size",1000) ##large than batch size
        self.avg_reward_len = kwargs.get("avg_reward_len",100)
        self.min_episode_reward =  kwargs.get("min_episode_reward",-21)
        self.seed = kwargs.get("seed",42)
        self.batches_per_epoch = kwargs.get("batches_per_epoch",1000)
        self.n_steps = kwargs.get("n_steps",1)

        super().__init__()
        self.env = env
        self.env.seed(self.seed)
        self.test_env = test_env
        self.obs_shape = self.env.observation_space.shape
        self.n_actions = self.env.action_space.n
        # Model Attributes
        self.buffer = None
        self.dataset = None

        self.net = None
        self.target_net = None

        self.build_networks()

        self.agent = ValueAgent(
            self.net,
            eps_start=self.eps_start,
            eps_end=self.eps_end,
            eps_frames=self.eps_last_frame,
        )

        self.save_hyperparameters()
        # Metrics
        self.total_episode_steps = [0]
        self.total_rewards = [0]
        self.done_episodes = 0
        self.total_steps = 0

        for _ in range(self.avg_reward_len):
            self.total_rewards.append(torch.tensor(self.min_episode_reward, device=self.device))

        self.avg_rewards = float(np.mean(self.total_rewards[-self.avg_reward_len :]))
        ##初始状态
        self.state = self.env.reset()

        self.action_idx_list = self.get_valid_action_space()
    def get_valid_action_space(self):
        actions = self.env.action.actions
        #pos_list=[[True,True],[True,False],[False,True],[False,False],no open]
        n= self.n_actions
        action_idx_list = [list(range(n)),list(range(n)),list(range(n)),list(range(n)),list(range(n))]
        for idx,item in enumerate(actions):
            ##无空不可平空
            if item and item[0] == Direction.LONG and item[1] == Offset.CLOSE:
                action_idx_list[1].remove(idx)
                action_idx_list[3].remove(idx)
            ##无多不可平多
            if item and item[0] == Direction.SHORT and item[1] == Offset.CLOSE:
                action_idx_list[2].remove(idx)
                action_idx_list[3].remove(idx)
            if item and item[1] == Offset.OPEN:
                action_idx_list[4].remove(idx)
        return action_idx_list
    def get_current_action_space(self):
        if self.env.portfolio.occupy_rate > self.env.action.limit_order_margin_rate:
            return self.action_idx_list[4]
        elif self.env.portfolio.long_pos > self.env.min_volume:
            if self.env.portfolio.short_pos > self.env.min_volume:
                return self.action_idx_list[0]
            else:
                return self.action_idx_list[1]
        else:
            if self.env.portfolio.short_pos > self.env.min_volume:
                return self.action_idx_list[2]
            else:
                return self.action_idx_list[3]

    def run_n_episodes(self, env, n_epsiodes: int = 1, epsilon: float = 1.0) -> List[int]:
        """Carries out N episodes of the environment with the current agent.
        Args:
            env: environment to use, either train environment or test environment
            n_epsiodes: number of episodes to run
            epsilon: epsilon value for DQN agent
        """
        total_rewards = []

        for _ in range(n_epsiodes):
            episode_state = env.reset()
            done = False
            episode_reward = 0

            while not done:
                self.agent.epsilon = epsilon
                action_space = self.get_current_action_space()
                action = self.agent(episode_state,action_space, self.device)
                next_state, reward, done, _ = env.step(action[0])
                episode_state = next_state
                episode_reward += reward

            total_rewards.append(episode_reward)

        return total_rewards
    def populate(self, warm_start: int) -> None:
        """Populates the buffer with initial experience."""
        if warm_start > 0:
            self.state = self.env.reset()

            for _ in range(warm_start):
                self.agent.epsilon = 1.0
                action_space = self.get_current_action_space()
                action = self.agent(self.state,action_space, self.device)
                next_state, reward, done, _ = self.env.step(action[0])
                exp = Experience(state=self.state, action=action[0], reward=reward, done=done, new_state=next_state)
                self.buffer.append(exp)
                self.state = next_state

                if done:
                    self.state = self.env.reset()
    def build_networks(self) -> None:
        """Initializes the DQN train and target networks."""
        ##input feature_dim = input_channel,size
        self.net = CNN(self.obs_shape[1],self.obs_shape[0],(self.obs_shape[1]+self.n_actions)//2, self.n_actions)
        self.target_net = CNN(self.obs_shape[1],self.obs_shape[0],(self.obs_shape[1]+self.n_actions)//2, self.n_actions)
    def forward(self, x: Tensor) -> Tensor:
        """Passes in a state x through the network and gets the q_values of each action as an output.
        Args:
            x: environment state
        Returns:
            q values
        """
        output = self.net(x)
        return output
    def train_batch(
        self,
    ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]:
        """Contains the logic for generating a new batch of data to be passed to the DataLoader.
        Returns:
            yields a Experience tuple containing the state, action, reward, done and next_state.
        """
        episode_reward = 0
        episode_steps = 0

        while True:
            self.total_steps += 1
            action_space = self.get_current_action_space()
            action = self.agent(self.state, action_space, self.device)

            next_state, r, is_done, _ = self.env.step(action[0])

            episode_reward += r
            episode_steps += 1

            exp = Experience(state=self.state, action=action[0], reward=r, done=is_done, new_state=next_state)

            self.agent.update_epsilon(self.global_step)
            self.buffer.append(exp)
            self.state = next_state

            if is_done:
                print("epsiode end")
                print(self.env.portfolio.net_capital)
                self.done_episodes += 1
                self.total_rewards.append(episode_reward)
                self.total_episode_steps.append(episode_steps)
                self.avg_rewards = float(np.mean(self.total_rewards[-self.avg_reward_len :]))
                self.state = self.env.reset()
                episode_steps = 0
                episode_reward = 0

            states, actions, rewards, dones, new_states = self.buffer.sample(self.batch_size)

            for idx, _ in enumerate(dones):
                yield states[idx], actions[idx], rewards[idx], dones[idx], new_states[idx]

            # Simulates epochs
            if self.total_steps % self.batches_per_epoch == 0:
                break

    def training_step(self, batch: Tuple[Tensor, Tensor], _) -> OrderedDict:
        """Carries out a single step through the environment to update the replay buffer. Then calculates loss
        based on the minibatch recieved.
        Args:
            batch: current mini batch of replay data
            _: batch number, not used
        Returns:
            Training loss and log metrics
        """

        # calculates training loss
        loss = dqn_loss(batch, self.net, self.target_net, self.gamma,torch.nn.SmoothL1Loss())

        if self._use_dp_or_ddp2(self.trainer):
            loss = loss.unsqueeze(0)

        # Soft update of target network
        if self.global_step % self.sync_rate == 0:
            self.target_net.load_state_dict(self.net.state_dict())

        self.log_dict(
            {
                "total_reward": self.total_rewards[-1],
                "avg_reward": self.avg_rewards,
                "train_loss": loss,
                "episodes": self.done_episodes,
                "episode_steps": self.total_episode_steps[-1],
            }
        )

        return OrderedDict(
            {
                "loss": loss,
                "avg_reward": self.avg_rewards,
            }
        )

    def test_step(self, *args, **kwargs) -> Dict[str, Tensor]:
        """Evaluate the agent for 10 episodes."""
        test_reward = self.run_n_episodes(self.test_env, 1, 0)
        avg_reward = sum(test_reward) / len(test_reward)
        return {"test_reward": avg_reward}

    def test_epoch_end(self, outputs) -> Dict[str, Tensor]:
        """Log the avg of the test results."""
        rewards = [x["test_reward"] for x in outputs]
        avg_reward = sum(rewards) / len(rewards)
        self.log("avg_test_reward", avg_reward)
        return {"avg_test_reward": avg_reward}

    def configure_optimizers(self) -> List[Optimizer]:
        """Initialize Adam optimizer."""
        optimizer = optim.Adam(self.net.parameters(), lr=self.learning_rate)
        return [optimizer]

    def _dataloader(self) -> DataLoader:
        """Initialize the Replay Buffer dataset used for retrieving experiences."""
        self.buffer = MultiStepBuffer(self.replay_size, self.n_steps)
        self.populate(self.warm_start_size)

        self.dataset = ExperienceSourceDataset(self.train_batch)
        return DataLoader(dataset=self.dataset, batch_size=self.batch_size)

    def train_dataloader(self) -> DataLoader:
        """Get train loader."""
        return self._dataloader()

    def test_dataloader(self) -> DataLoader:
        """Get test loader."""
        return self._dataloader()
    @staticmethod
    def _use_dp_or_ddp2(trainer: Trainer) -> bool:
        return isinstance(trainer.training_type_plugin, (DataParallelPlugin, DDP2Plugin))


import pandas as pd
from time import perf_counter
from typing import Any, Dict, List, Optional, Set, Tuple
from mitosheet.ai.ai_utils import fix_final_dataframe_name, fix_up_missing_imports
from mitosheet.code_chunks.code_chunk import CodeChunk
from mitosheet.code_chunks.ai_transformation_code_chunk import AITransformationCodeChunk

from mitosheet.state import State
from mitosheet.step_performers.step_performer import StepPerformer
from mitosheet.step_performers.utils.utils import get_param
from mitosheet.ai.recon import exec_and_get_new_state_and_result

class AITransformationStepPerformer(StepPerformer):
    """
    Allows you to execute an arbitrary chunk of code, generated by an AI.
    """

    @classmethod
    def step_version(cls) -> int:
        return 1

    @classmethod
    def step_type(cls) -> str:
        return 'ai_transformation'

    @classmethod
    def execute(cls, prev_state: State, params: Dict[str, Any]) -> Tuple[State, Optional[Dict[str, Any]]]:
        # We don't use any of these parameters, but we keep them for clarity, and we don't want to remove them accidently
        user_input: str = get_param(params, 'user_input')
        prompt_version: str = get_param(params, 'prompt_version')
        prompt: str = get_param(params, 'prompt')
        completion: str = get_param(params, 'completion')
        edited_completion: str = get_param(params, 'edited_completion')

        pandas_start_time = perf_counter()
        post_state, last_line_value, frontend_result = exec_and_get_new_state_and_result(prev_state, edited_completion)
        pandas_processing_time = perf_counter() - pandas_start_time

        if isinstance(last_line_value, pd.DataFrame) or isinstance(last_line_value, pd.Series):
            final_code = fix_final_dataframe_name(edited_completion, post_state.df_names[-1], isinstance(last_line_value, pd.Series))
        else:
            final_code = edited_completion

        return post_state, {
            'pandas_processing_time': pandas_processing_time,
            'result': frontend_result,
            'final_code': final_code
        }

    @classmethod
    def transpile(
        cls,
        prev_state: State,
        post_state: State,
        params: Dict[str, Any],
        execution_data: Optional[Dict[str, Any]],
    ) -> List[CodeChunk]:
        return [
            AITransformationCodeChunk(
                prev_state,
                post_state,
                get_param(params, 'user_input'),
                get_param(execution_data if execution_data is not None else {}, 'final_code')
            )
        ]

    @classmethod
    def get_modified_dataframe_indexes(cls, params: Dict[str, Any]) -> Set[int]:
        return set() # NOTE: We act as though we modify all sheet indexes, as we can't easily figure out what we do modify

    @classmethod
    def get_created_dataframe_indexes(cls, params: Dict[str, Any]) -> Set[int]:
        return {-1} # NOTE: We act as though we create all sheet indexes, as we can't easily figure out what we do modify


from nba_api.stats.endpoints import teamyearbyyearstats, teamplayerdashboard, teamdashboardbylastngames, \
    teamdashboardbyshootingsplits
from nba_api.stats.static import teams

def get_team_id(teamName):
    nba_teams = teams.get_teams()

    selected_team = [team for team in nba_teams
                     if team['full_name'] == teamName][0]

    teamInfo = teamyearbyyearstats.TeamYearByYearStats(team_id=str(selected_team.get('id')))

    return teamInfo.get_data_frames()[0].TEAM_ID[0]

def get_team_info(teamName):
    team_id = get_team_id(teamName)

    team_stats = teamyearbyyearstats.TeamYearByYearStats(team_id=team_id).get_data_frames()[0]

    season_played = []
    for season in team_stats['YEAR']:
        season_played.append(season)

    game_played = []
    for gp in team_stats['GP']:
        game_played.append(gp)

    wins = []
    for win in team_stats['WINS']:
        wins.append(win)

    losses = []
    for loss in team_stats['LOSSES']:
        losses.append(loss)

    win_percentage = []
    for win_pct in team_stats['WIN_PCT']:
        win_percentage.append(win_pct)

    conf_rank = []
    for _ in team_stats['CONF_RANK']:
        conf_rank.append(_)

    po_wins = []
    for powins in team_stats['PO_WINS']:
        po_wins.append(powins)

    po_losses = []
    for polosses in team_stats['PO_LOSSES']:
        po_losses.append(polosses)

    # N/A - not in final, FINALS APPEARANCE - lost in final, LEAGUE CHAMPION - won in final.
    final_appearance = []
    for _ in team_stats['NBA_FINALS_APPEARANCE']:
        final_appearance.append(_)

    po_gp = [[] for _ in po_wins]
    for i in range(len(po_gp)):
        po_gp[i] = po_wins[i] + po_losses[i]

    latest_season = season_played[-1]
    team_dashboard = teamplayerdashboard.TeamPlayerDashboard(team_id=team_id, season=latest_season).get_data_frames()[1]

    player_name = []
    for player in team_dashboard['PLAYER_NAME']:
        player_name.append(player)

    team_lastgame = teamdashboardbylastngames.TeamDashboardByLastNGames(
        team_id=team_id, season=latest_season).get_data_frames()[1]

    last_5_games = ['Last 5 games']
    w = []
    for win in team_lastgame['W']:
        w.append(win)

    l = []
    for lose in team_lastgame['L']:
        l.append(lose)

    w_pct = []
    for win_pct in team_lastgame['W_PCT']:
        w_pct.append(win_pct)

    # field goal attempt
    fga = []
    for FGA in team_lastgame['FGA']:
        fga.append(FGA)
    # field goal made
    fgm = []
    for FGM in team_lastgame['FGM']:
        fgm.append(FGM)

    fg_pct = []
    for FG_pct in team_lastgame['FG_PCT']:
        fg_pct.append(FG_pct)

    reb = []
    for rebound in team_lastgame['REB']:
        reb.append(rebound)

    ast = []
    for assist in team_lastgame['AST']:
        ast.append(assist)
    # turnovers
    tov = []
    for turnover in team_lastgame['TOV']:
        tov.append(turnover)

    stl = []
    for steal in team_lastgame['STL']:
        stl.append(steal)

    blk = []
    for block in team_lastgame['BLK']:
        blk.append(block)

    pts = []
    for point in team_lastgame['PTS']:
        pts.append(point)
    # team efficiency
    plus_minus = []
    for plusandminus in team_lastgame['PLUS_MINUS']:
        plus_minus.append(plusandminus)

    team_shooting = teamdashboardbyshootingsplits.TeamDashboardByShootingSplits(
        team_id=team_id, season=latest_season).get_data_frames()[3]

    shot_area = []
    for values in team_shooting['GROUP_VALUE']:
        shot_area.append(values)

    # field goal attempt
    fga_area = []
    for FGA in team_shooting['FGA']:
        fga_area.append(FGA)
    # field goal made
    fgm_area = []
    for FGM in team_shooting['FGM']:
        fgm_area.append(FGM)

    fg_pct_area = []
    for FG_pct in team_shooting['FG_PCT']:
        fg_pct_area.append(FG_pct)

    return [season_played, game_played, wins, losses, win_percentage, conf_rank, po_gp, po_wins, po_losses,
        final_appearance], \
        [player_name], \
        [last_5_games, w, l, w_pct, fga, fgm, fg_pct, pts, ast, reb, stl, blk, tov, plus_minus], \
        [shot_area, fga_area, fgm_area, fg_pct_area]

def get_team_info_using_teamid(teamID):
    team_stats = teamyearbyyearstats.TeamYearByYearStats(teamID).get_data_frames()[0]

    season_played = []
    for season in team_stats['YEAR']:
        season_played.append(season)

    game_played = []
    for gp in team_stats['GP']:
        game_played.append(gp)

    wins = []
    for win in team_stats['WINS']:
        wins.append(win)

    losses = []
    for loss in team_stats['LOSSES']:
        losses.append(loss)

    win_percentage = []
    for win_pct in team_stats['WIN_PCT']:
        win_percentage.append(win_pct)

    conf_rank = []
    for _ in team_stats['CONF_RANK']:
        conf_rank.append(_)

    po_wins = []
    for powins in team_stats['PO_WINS']:
        po_wins.append(powins)

    po_losses = []
    for polosses in team_stats['PO_LOSSES']:
        po_losses.append(polosses)

    # N/A - not in final, FINALS APPEARANCE - lost in final, LEAGUE CHAMPION - won in final.
    final_appearance = []
    for _ in team_stats['NBA_FINALS_APPEARANCE']:
        final_appearance.append(_)

    po_gp = [[] for _ in po_wins]
    for i in range(len(po_gp)):
        po_gp[i] = po_wins[i] + po_losses[i]

    return [season_played, game_played, wins, losses, win_percentage, conf_rank, po_gp, po_wins, po_losses,
            final_appearance]


from flask import Flask, request

app = Flask(__name__)

@app.route('/index', methods=['get', 'post'])
def index():
    return 'index page'

@app.route('/upload', methods=['post'])
def upload():
    '''接收文件'''
    file_obj = request.files.get('pic')
    if not file_obj:
        return '未上传文件'

    # 第一种：保存文件
    # f = open('./demo.jpg', 'wb')
    # data = file_obj.read()
    # f.write(data)
    # f.close()
    # 第二种保存文件
    file_obj.save('./demo1.png')

    return '上传成功'

if __name__ == '__main__':
    app.run(debug=True)


import logging

from urwid import BoxAdapter, LineBox, Text, connect_signal

from subiquitycore.models.network import WLANConfig
from subiquitycore.ui.buttons import cancel_btn, menu_btn
from subiquitycore.ui.container import ListBox, Pile, WidgetWrap
from subiquitycore.ui.form import Form, PasswordField, StringField
from subiquitycore.ui.stretchy import Stretchy
from subiquitycore.ui.utils import Color, Padding, disabled

log = logging.getLogger("subiquitycore.ui.views.network_configure_wlan_interface")

class NetworkList(WidgetWrap):
    def __init__(self, parent, ssids):
        self.parent = parent
        button = cancel_btn(_("Cancel"), on_press=self.do_cancel)
        ssid_list = [
            menu_btn(label=ssid, on_press=self.do_network) for ssid in ssids if ssid
        ]
        p = Pile([BoxAdapter(ListBox(ssid_list), height=10), Padding.fixed_10(button)])
        box = LineBox(p, title=_("Select a network"))
        super().__init__(box)

    def do_network(self, sender):
        self.parent.form.ssid.value = sender.label
        self.parent.parent.remove_overlay()

    def do_cancel(self, sender):
        self.parent.parent.remove_overlay()

class WLANForm(Form):
    ok_label = _("Save")

    ssid = StringField(caption="Network Name:")
    psk = PasswordField(caption="Password:")

    def validate_psk(self):
        psk = self.psk.value
        if len(psk) == 0:
            return
        elif len(psk) < 8:
            return "Password must be at least 8 characters long if present"
        elif len(psk) > 63:
            return "Password must be less than 63 characters long"

class NetworkConfigureWLANStretchy(Stretchy):
    def __init__(self, parent, dev_info):
        self.parent = parent
        self.dev_info = dev_info
        title = _("Network interface {nic} WIFI configuration").format(
            nic=dev_info.name
        )

        self.form = WLANForm()

        connect_signal(self.form, "submit", self.done)
        connect_signal(self.form, "cancel", self.cancel)

        if self.dev_info.wlan.config.ssid:
            self.form.ssid.value = self.dev_info.wlan.config.ssid
        if self.dev_info.wlan.config.psk:
            self.form.psk.value = self.dev_info.wlan.config.psk

        self.ssid_row = self.form.ssid._table
        self.psk_row = self.form.psk._table
        self.ssid_row.bind(self.psk_row)

        self.error = Text("")

        self.inputs = Pile(self._build_iface_inputs())

        widgets = [
            self.inputs,
            Padding.center_79(Color.info_error(self.error)),
            self.form.buttons,
        ]
        super().__init__(title, widgets, 0, 0)

    def show_ssid_list(self, sender):
        self.parent.show_overlay(
            NetworkList(self, self.dev_info.wlan.visible_ssids), width=60
        )

    def start_scan(self, sender):
        fp = self.inputs.focus_position - 1
        while not self.inputs.contents[fp][0].selectable():
            fp -= 1
        self.inputs.focus_position = fp
        self.parent.controller.start_scan(self.dev_info.name)

    def _build_iface_inputs(self):
        visible_ssids = self.dev_info.wlan.visible_ssids
        if len(visible_ssids) > 0:
            networks_btn = menu_btn(
                "Choose a visible network", on_press=self.show_ssid_list
            )
        else:
            networks_btn = disabled(menu_btn("No visible networks"))

        scan_state = self.dev_info.wlan.scan_state
        if not scan_state:
            scan_btn = menu_btn("Scan for networks", on_press=self.start_scan)
        elif scan_state.startswith("error"):
            self.error.set_text("scan failed: %s" % (scan_state,))
            scan_btn = disabled(menu_btn("Scanning for networks failed"))
        else:
            scan_btn = disabled(menu_btn("Scanning for networks"))

        warning = "Only open or WPA2/PSK networks are supported at this time."
        col = [
            Text(warning),
            Text(""),
            self.ssid_row,
            Text(""),
            Padding.fixed_32(networks_btn),
            Padding.fixed_32(scan_btn),
            Text(""),
            self.psk_row,
        ]
        return col

    def update_link(self, dev_info):
        if dev_info.name != self.dev_info.name:
            return
        self.dev_info = dev_info
        self.inputs.contents = [
            (obj, ("pack", None)) for obj in self._build_iface_inputs()
        ]

    def done(self, sender):
        if self.form.ssid.value:
            ssid = self.form.ssid.value
        else:
            ssid = None
        if self.form.psk.value:
            psk = self.form.psk.value
        else:
            psk = None
        self.parent.controller.set_wlan(
            self.dev_info.name, WLANConfig(ssid=ssid, psk=psk)
        )
        self.parent.update_link(self.dev_info)
        self.parent.remove_overlay()

    def cancel(self, sender=None):
        self.parent.remove_overlay()


import numpy as numpy
import csv
from sklearn.neighbors import KNeighborsClassifier
import pandas
from sklearn.preprocessing import StandardScaler

file_name = 'student-mat-nograde3.csv'

data_frame = pandas.read_csv(file_name)
data_frame = pandas.get_dummies(data_frame)
print(data_frame)
#raw_data = open(file_name, 'rt')
numpy_array = data_frame.values

#using scaler
scaler = StandardScaler()
scaler.fit(numpy_array)
numpy_array = scaler.transform(numpy_array)

print(numpy_array)

file_name = 'student-math.csv'
one_row = []
raw_data = open(file_name, 'rt')

with open(file_name) as f:
    reader = csv.reader(raw_data, delimiter=',')
    next(reader)
    data_y = []
    for i, row in enumerate(reader):
        if int(row[len(row) - 1]) > 15:
            data_y.append(1)
        else:
            data_y.append(0)
        #data_y.append(row[len(row) - 1])
    y = list(data_y)
    data_y = numpy.array(y).astype('int')

neigh = KNeighborsClassifier(n_neighbors=4)
print(data_y)
print(data_y.shape)
neigh.fit(numpy_array, data_y)
print(data_y)
print(neigh.score(numpy_array, data_y))


import stripe
import frappe
import datetime

# price IDS
@frappe.whitelist(allow_guest=True)
def hasActiveSubscription(*args, **kwargs):
    invalidate_cache = True if "invalidate_cache" in kwargs else False
    site = kwargs.get("site", frappe.local.site)
    if not site:
        return False
    frappe.utils.execute_in_shell("bench --site {} clear-cache".format(site))
    frappe.utils.execute_in_shell("bench --site {} clear-website-cache".format(site))
    country = frappe.get_site_config(site_path=site)["country"]
    customer_id = frappe.get_site_config(site_path=site)["customer_id"]
    stripe_manager = StripeSubscriptionManager(country=country)

    if invalidate_cache or (not frappe.conf.has_subscription):
        hasSubscription = (
            "yes" if stripe_manager.has_valid_site_subscription(customer_id) else "no"
        )
        # save site expiry date in site_config

        command = "bench --site {} set-config has_subscription {}".format(
            site, hasSubscription
        )
        frappe.utils.execute_in_shell(command)
    result = True if frappe.conf.has_subscription == "yes" else False
    return result

def get_room(customer_id):
    try:
        site = frappe.db.get_list(
            "SaaS sites",
            filters={"cus_id": customer_id},
            fields=["site_name"],
            ignore_permissions=True,
        )[0]["site_name"]
        return f"{site}:website"
    except Exception as e:
        print("error", e)

def get_site(customer_id):
    site = frappe.db.get_list(
        "SaaS sites",
        filters={"cus_id": customer_id},
        fields=["site_name"],
        ignore_permissions=True,
    )
    return site[0]["site_name"] if len(site) > 0 else None

class StripeSubscriptionManager:
    def __init__(self, country=""):
        self.api_key = frappe.conf.STRIPE_SECRET_KEY
        self.endpoint_secret = frappe.conf.STRIPE_ENDPOINT_SECRET
        if country:
            self.region = country
        else:
            self.region = frappe.conf.country or "US"
        if self.region == "IN":
            self.api_key = frappe.conf.STRIPE_SECRET_KEY_IN

            self.endpoint_secret = frappe.conf.STRIPE_ENDPOINT_SECRET_IN
        self.plan_to_product_id = frappe.conf.stripe_prices["US"]["products"]

        self.trial_price_id = frappe.conf.stripe_prices["US"]["trial_price_id"]
        self.trial_product = "ONEHASH_PRO"
        print("region", self.region)
        if self.region == "IN":
            print("setting plan to product id for IN")
            self.plan_to_product_id = frappe.conf.stripe_prices["IN"]["products"]
            self.trial_price_id = frappe.conf.stripe_prices["IN"]["trial_price_id"]
        print("plan to product id", self.plan_to_product_id)
        self.onehas_subscription_product_ids = [
            x for x in self.plan_to_product_id.values()
        ]
        print("stripe api key", self.api_key)
        stripe.api_key = self.api_key

    def plan_id_to_product(self):
        return {v: k for k, v in self.plan_to_product_id.items()}

    def get_subscriptions(self, customer_id):
        return stripe.Subscription.list(customer=customer_id)

    def getSession(self, session_id, expand=[]):
        return stripe.checkout.Session.retrieve(session_id, expand=expand)

    def create_customer(self, site_name, email, fname, lname, phone):
        return stripe.Customer.create(
            email=email,
            metadata={"full name": fname + " " + lname},
            name=site_name,
            phone=phone,
        )

    def get_current_onehash_price(self, customer_id):
        subscription = self.get_onehash_subscription(frappe.conf.customer_id)
        return subscription["plan"]["id"]

    def end_trial(self, customer_id):
        alls = stripe.Customer.list()
        subscriptions = stripe.Subscription.list(customer=customer_id)
        for subscription in subscriptions:
            if subscription["status"] == "trialing":
                stripe.Subscription.delete(subscription["id"])

    def start_free_trial_of_site(self, customer_id):
        return stripe.Subscription.create(
            customer=customer_id,
            items=[{"price": self.trial_price_id}],
            payment_settings={"save_default_payment_method": "on_subscription"},
            trial_settings={"end_behavior": {"missing_payment_method": "pause"}},
            metadata={"plan": "ONEHASH_PRO"},
            trial_period_days=14,
        )

    def has_valid_site_subscription(self, cus_id):
        return self.get_onehash_subscription(cus_id) != "NONE"

    def create_new_purchase_session(self, customer_id, price_id, subdomain):
        success_url = (
            "http://"
            + subdomain
            + "."
            + (
                frappe.conf.domain
                if frappe.conf.domain != "localhost"
                else frappe.conf.domain + ":8000"
            )
            + "/plans?payment_success=True"
        )
        session = stripe.checkout.Session.create(
            success_url=success_url,
            mode="subscription",
            line_items=[
                {
                    "price": price_id,
                    "quantity": 1,
                }
            ],
            client_reference_id=subdomain,
            customer=customer_id,
            metadata={"del_trial": True},
        )
        return session.url

    # def create_checkout_session(self,customer_id,plan,success_url,cancel_url):
    def get_customer_details(self, customer_id):
        customer = stripe.Customer.retrieve(customer_id, expand=["subscriptions"])
        return customer

    def get_current_onehash_product_id(self, customer_id):
        customer = self.get_customer_details(customer_id)
        product_id = None
        for subscription in customer["subscriptions"]["data"]:
            if (
                subscription["status"] in ["active", "trialing"]
                and subscription["plan"]["product"]
                and subscription["plan"]["product"]
                in self.onehas_subscription_product_ids
            ):
                product_id = subscription["plan"]["product"]
                break
        return product_id

    def get_current_onehash_product(self, customer_id):
        subscriptions = stripe.Subscription.list(customer=customer_id)
        product = None
        for subscription in subscriptions["data"]:
            if (
                subscription["status"] in ["active", "trialing"]
                and subscription["plan"]["product"]
                and subscription["plan"]["product"]
                in self.onehas_subscription_product_ids
            ):
                product = subscription["plan"]["product"]
                break
        return stripe.Product.retrieve(product)

    def hadle_subscription_deleted(self, event):
        hasActiveSubscription(
            invalidate_cache=True, site=get_site(event["data"]["object"]["customer"])
        )

    def has_valid_subscription_v2(self, customer_id, plan):
        # if plan == "ONEHASH"
        # check if the customer has any product in subscription which product id starts with ONEHASH
        # if yes then return true else false
        if not customer_id:
            return False
        customer = self.get_customer_details(customer_id)
        if "subscriptions" not in customer:
            return False
        if plan == "ONEHASH":
            for subscription in customer["subscriptions"]["data"]:
                if (
                    subscription["status"] in ["active", "trialing"]
                    and subscription["plan"]["product"]
                    and subscription["plan"]["product"]
                    in self.onehas_subscription_product_ids
                ):
                    return True
            return False
        else:
            for subscription in customer["subscriptions"]["data"]:
                if (
                    subscription["status"] in ["active", "trialing"]
                    and subscription["plan"]["product"]
                    and subscription["plan"]["product"] == self.plan_to_product_id[plan]
                ):
                    return True
            return False

    def get_onehash_subscription(self, customer_id):
        print("getting onehash subscription for customer", customer_id)
        if not customer_id:
            return "NONE"
        subscriptions = stripe.Subscription.list(customer=customer_id)
        for subscription in subscriptions["data"]:
            print("subscription status", subscription["status"])
            print("subscription plan", subscription["plan"]["product"])
            print("product ids", self.onehas_subscription_product_ids)
            print(
                "subscription in onehash",
                subscription["plan"]["product"] in self.onehas_subscription_product_ids,
            )
            if (
                subscription["status"] in ["active", "trialing"]
                and subscription["plan"]["product"]
                and (
                    subscription["plan"]["product"]
                    in self.onehas_subscription_product_ids
                )
            ):
                return subscription
        return "NONE"

    def cancel_onehash_subscription(self, customer_id):
        current_sub = self.get_onehash_subscription(customer_id)
        if current_sub != "NONE":
            stripe.Subscription.delete(current_sub["id"])

    def upgrade_subscription(self, customer_id, new_price_id, subdomain):
        # this will upgrade the subscription to new price id and try to charge the customer immediately
        subscription = self.get_onehash_subscription(customer_id)
        # check is customer has any pending invoice if yes then return "PENDING_INVOICE"
        # check is customer has any pending update if yes then return "PENDING_UPDATE"
        # check is customer has any unpaid invoice if yes then return "UNPAID_INVOICE"
        # check is customer has no payment method if yes then return "NO_PAYMENT_METHOD"
        customer = self.get_customer_details(customer_id)
        invoices = stripe.Invoice.list(customer=customer_id)
        for invoice in invoices["data"]:
            if invoice["status"] != "paid" and invoice["status"] != "void":
                return "PENDING_INVOICE"
        customer_payment_methods = stripe.PaymentMethod.list(
            customer=customer_id, type="card"
        )

        if len(customer_payment_methods["data"]) == 0:
            return "NO_PAYMENT_METHOD"
        # remove any pending update
        if customer["subscriptions"]["data"][0]["pending_update"]:
            return "PENDING_UPDATE"

        try:
            response = stripe.Subscription.modify(
                subscription.id,
                payment_behavior="pending_if_incomplete",
                proration_behavior="always_invoice",
                items=[
                    {
                        "id": subscription["items"]["data"][0].id,
                        "price": new_price_id,
                    }
                ],
            )

        except Exception as e:
            return "TECHNICAL_ERROR"
        if "pending_update" in response and response["pending_update"]:
            return "PENDING_UPDATE"

        else:
            # success
            return "SUCCESS"

    def handle_payment_intent_failed(self, event):
        customer_id = event["data"]["object"]["customer"]
        hasActiveSubscription(
            invalidate_cache=True, site=get_site(event["data"]["object"]["customer"])
        )
        frappe.publish_realtime("payment_failed", room=get_room(customer_id))

    def handle_payment_intent_action_required(self, event):
        #   payment_intent_id = "pi_1NUkv1CwmuPVDwVySsT5gUDX"
        payment_intent_id = event["data"]["object"]["payment_intent"]
        payment_intent = stripe.PaymentIntent.retrieve(payment_intent_id)
        client_secret = payment_intent["client_secret"]
        frappe.publish_realtime(
            "requires_payment_action",
            message={"client_secret": client_secret},
            room=get_room(event["data"]["object"]["customer"]),
        )
        # after 3 minute check if payment is done , if not dome then void this transaction
        from threading import Timer

        invoice_id = event["data"]["object"]["id"]

        def void_payment(invoice_id):
            payment_intent = stripe.PaymentIntent.retrieve(invoice_id)
            if payment_intent["status"] == "requires_payment_action":
                stripe.Invoice.void_invoice(invoice_id)

        t = Timer(30 * 60, void_payment, (invoice_id))
        t.start()

    def handle_subscription_updated(self, event):
        hasActiveSubscription(
            invalidate_cache=True, site=get_site(event["data"]["object"]["customer"])
        )

    def handle_checkout_session_completed(self, event):
        session_metadata = event["data"]["object"]["metadata"]
        customer_id = event["data"]["object"]["customer"]
        if "del_trial" in session_metadata and session_metadata["del_trial"] == "True":
            self.end_trial(customer_id)

    def get_product(self, product_id):
        return stripe.Product.retrieve(product_id)

    def handle_invoice_failed(self, event):
        hasActiveSubscription(
            invalidate_cache=True, site=get_site(event["data"]["object"]["customer"])
        )

    def handle_invoice_paid(self, event):
        customer_id = event["data"]["object"]["customer"]
        sub_id = event["data"]["object"]["subscription"]
        subscription = stripe.Subscription.retrieve(sub_id, expand=["latest_invoice"])
        product_id = subscription["items"]["data"][0]["price"]["product"]
        price_id = subscription["items"]["data"][0]["price"]["id"]
        # invoice paid
        if product_id in self.onehas_subscription_product_ids:
            site_name = get_site(customer_id)
            if not site_name:
                return
            if price_id != frappe.conf.price_id:
                # end tria
                # fetch the site_name from the database - saved payment intent id
                print("updating onehash subscription for site", site_name)
                fulfilOneHashUpdate(
                    self.onehas_subscription_product_ids,
                    product_id,
                    price_id,
                    site_name,
                )
                # call payment success on that site
            command = "bench --site {} set-config has_subscription {}".format(
                site_name, "yes"
            )
            frappe.utils.execute_in_shell(command)
            frappe.publish_realtime("payment_success", room=get_room(customer_id))

def test2():
    hasActiveSubscription(invalidate_cache=True, site="dlwkef.localhost")

def fulfilOneHashUpdate(pids, product_id, price_id, site_name):
    # pass the checkout session
    if product_id in pids:
        # handle onehash subscription
        product = stripe.Product.retrieve(product_id)
        user_limit = 10
        plan_name = ""
        if product.name == "OneHash Pro":
            user_limit = 100000
            plan_name = "ONEHASH_PRO"
        elif product.name == "OneHash Starter":
            user_limit = 10
            plan_name = "ONEHASH_STARTER"
        else:
            user_limit = 30
            plan_name = "ONEHASH_PLUS"
        command_to_set_limit = (
            "bench --site {site_name} set-config  max_users {user_limit}".format(
                site_name=site_name, user_limit=user_limit
            )
        )
        command_to_set_plan = "bench --site {site_name} set-config  plan {plan}".format(
            site_name=site_name, plan=plan_name
        )
        command_to_set_price_id = (
            "bench --site {site_name} set-config  price_id {price_id}".format(
                site_name=site_name, price_id=price_id
            )
        )
        frappe.utils.execute_in_shell(command_to_set_limit)
        frappe.utils.execute_in_shell(command_to_set_plan)
        frappe.utils.execute_in_shell(command_to_set_price_id)

# unit testing to do.

# test if new signups are being made - site has a trial subscription and  correspnding user limits are being setpayment_intent = stripe.PaymentIntent.retrieve(payment_intent_id)

def checkCorrectEmailAddressWithRegex(email):
    if email == "":
        return False
    import re

    return re.match(r"[^@]+@[^@]+\.[^@]+", email)

def updateDoctypeWithDocs(new_doc, doctype_name):
    doc = frappe.get_doc(doctype_name, new_doc.name)
    doc.update(new_doc)
    doc.save()
    return doc


from keras import callbacks

def callback_for_training(tf_log_dir_name='rfrl-network',patience_lr=10,snapshot_name=None):
    cb = [None] * 3
    """
    Tensorboard log callback
    """
    tb = callbacks.TensorBoard(log_dir=tf_log_dir_name, histogram_freq=0)
    cb[0]= tb

    """
    Model Checkpointer
    """
    checkpointer = callbacks.ModelCheckpoint(filepath=snapshot_name+".{epoch:02d}-{val_classifier_accuracy:.2f}.hdf5",
                            verbose=0,
                            monitor='val_classifier_accuracy')
    cb[1] = checkpointer

    """
    Reduce Learning Rate
    """
    reduce_lr_loss = callbacks.ReduceLROnPlateau(monitor='val_classifier_loss', factor=0.1, patience=8, verbose=1, min_lr=1e-8, mode='auto')
    cb[2] = reduce_lr_loss

    return cb


import tensorflow as tf
import numpy as np
from tensorflow.python.keras.backend import dtype
from util import fid as FID
from util import restrict_gpu, PreprocessModel
from data import celeba_local as CELEBA

from util import restrict_gpu, PreprocessModel
from networks.STARGAN import StarGAN as SG_BASE
from networks.STARGAN_multicycle import StarGAN_MultiCycle as SG_MC
from networks.STARGAN_multitrain import StarGAN_MultiCycle as SG_MT

import os

def main(use_lpips=False):

    #restrict_gpu.chooseOneGPU(int(input("Input the ID of the GPU you'd Like to choose: ")))
    AUTOTUNE = tf.data.AUTOTUNE

    load_dir = "D:/celeba/"
    #load_dir = '/media/Data1/CELEBA/'
    labels = ["Blond_Hair", "Brown_Hair", "Black_Hair", "Male", "Young"]
    label_mask = tf.constant([1.0 if 'hair' in label.lower() else 0.0 for label in labels])

    dataset, test_dataset, val_dataset = CELEBA.load_celeba(load_dir, labels, tt_split=True)

    BATCH_SIZE = 16
    IMG_WIDTH = 178
    IMG_HEIGHT = 218

    RESCALE_DIM = 128
    EPOCHS=20

    preprocess_model = PreprocessModel.get_preprocess_model(IMG_WIDTH, RESCALE_DIM, random_flip=False)
    fid_model = FID.get_model([RESCALE_DIM, RESCALE_DIM, 3])
    model_idx = int(input("Input the Num of the model you want to eval on (1 for Stargan, 2 for SG_MC, 3 for SG_MT): "))

    if model_idx < 0:
        dataset = dataset.shard(80, 1)
        val_dataset = val_dataset.shard(80, 1)
        test_dataset= test_dataset.shard(80, 1)
        model_idx = -model_idx

    if model_idx == 1:
        model_date = "12_06/16/"
        model_folder = './FINAL_OUTPUT/starGAN_celeba/'
        model = SG_BASE(
            [RESCALE_DIM, RESCALE_DIM, 3], len(labels),
            model_folder,
            time_created=model_date,
            preprocess_model=preprocess_model,
            restore_model_from_checkpoint=False
            )
    elif model_idx == 2:
        model_date = "12_08/18/"
        model_folder = './FINAL_OUTPUT/sg_multi_celeba/'
        model = SG_MC(
            [RESCALE_DIM, RESCALE_DIM, 3], len(labels),
            model_folder,
            time_created=model_date,
            preprocess_model=preprocess_model,
            restore_model_from_checkpoint=False
            )
    elif model_idx == 3:
        model_date = "12_11/22/"
        model_folder = './FINAL_OUTPUT/sg_multitrain_celeba/'
        model = SG_MT(
            [RESCALE_DIM, RESCALE_DIM, 3], len(labels),
            model_folder,
            time_created=model_date,
            preprocess_model=preprocess_model,
            restore_model_from_checkpoint=False
            )
    else:
        raise(ValueError("Model ID out of Range!"))

    if use_lpips:
        import util.lpips as lpips
        lpips_model = lpips.get_lpips_model(RESCALE_DIM)

    def get_fids(base_dataset, n_cycles=0, test_lpips=True, save_imgs = False, save_to=model_folder, trailing_zeros=5):

        if save_imgs and not os.path.exists(save_to):
            os.makedirs(save_to)

        base_dataset = base_dataset.batch(BATCH_SIZE)

        if test_lpips:
            lpips_avg = tf.keras.metrics.Mean("lpips average", dtype=tf.float32)
            lpips_recon = tf.keras.metrics.Mean("lpips reconstructive average", dtype=tf.float32)

        base_acts = []
        predicted_acts = []
        reconstructed_acts = []
        label_accuracy = tf.keras.metrics.BinaryAccuracy("label_acc", dtype=tf.float32)
        recon_accuracy = tf.keras.metrics.BinaryAccuracy('recon_acc', dtype=tf.float32)
        for i, batch in base_dataset.enumerate():
            print(f"\rCalculating Activations for images {i*BATCH_SIZE} through {((i+1)*BATCH_SIZE ) - 1}", end="")
            images = batch['image']
            labels = batch['label']
            preprocessed = preprocess_model.predict(images)
            base_acts.append(fid_model.predict(preprocessed))
            predict_on = preprocessed
            for n in range(n_cycles):
                #print(i)
                cycle_labels = model.gen_target(labels)
                predict_on = model.gen.model.predict([predict_on, cycle_labels])
            inv_labels = model.gen_inv_target(labels, label_mask)
            predicted = tf.clip_by_value(model.gen.model.predict([predict_on, inv_labels]), -1.0, 1.0)
            predicted_acts.append(fid_model.predict(predicted))
            reconstructed = tf.clip_by_value(model.gen.model.predict([predict_on, labels]), -1.0, 1.0)
            reconstructed_acts.append(fid_model.predict(reconstructed))
            inv_pred_labels = tf.math.sigmoid(model.disc.model.predict(predicted)[1])
            recon_pred_labels = tf.math.sigmoid(model.disc.model.predict(reconstructed)[1])
            label_accuracy(inv_labels, inv_pred_labels)
            recon_accuracy(labels, recon_pred_labels)
            if save_imgs or save_imgs is tuple:
                if save_imgs is not tuple:
                    save_predicted = save_imgs
                    save_base = save_imgs
                else:
                    save_base, save_predicted = save_imgs
                for j, (image, predicted_image, reconstructed_image) in enumerate(zip(preprocessed*0.5+0.5, predicted*0.5+0.5, reconstructed*0.5+0.5)):
                    if save_base:
                        image = tf.clip_by_value(image, 0.0, 1.0)
                        image = tf.image.convert_image_dtype(image, tf.uint8)
                        encoded = tf.io.encode_jpeg(image, format='rgb')
                        tf.io.write_file(os.path.join(save_to, 'images/', ("image{:0"+str(trailing_zeros)+"d}.jpg").format(i*BATCH_SIZE+j)), encoded)
                    if save_predicted:
                        predicted_image = tf.clip_by_value(predicted_image, 0.0, 1.0)
                        predicted_image = tf.image.convert_image_dtype(predicted_image, tf.uint8)
                        predicted_encoded = tf.io.encode_jpeg(predicted_image, format='rgb')
                        tf.io.write_file(os.path.join(save_to, f'images_cycled_{n_cycles}/', ("image{:0"+str(trailing_zeros)+"d}.jpg").format(i*BATCH_SIZE+j)), predicted_encoded)
                        reconstructed_image = tf.clip_by_value(reconstructed_image, 0.0, 1.0)
                        reconstructed_image = tf.image.convert_image_dtype(reconstructed_image, tf.uint8)
                        reconstructed_encoded = tf.io.encode_jpeg(reconstructed_image, format='rgb')
                        tf.io.write_file(os.path.join(save_to, f'images_cycled_{n_cycles}_reconstructed/', ("image{:0"+str(trailing_zeros)+"d}.jpg").format(i*BATCH_SIZE+j)), reconstructed_encoded)
            if test_lpips:
                lpips_img = tf.cast(tf.image.convert_image_dtype((preprocessed*0.5+0.5), tf.uint8), tf.float32)
                lpips_predicted = tf.cast(tf.image.convert_image_dtype((predicted*0.5+0.5), tf.uint8), tf.float32)
                lpips_reconstructed = tf.cast(tf.image.convert_image_dtype((reconstructed*0.5+0.5), tf.uint8), tf.float32)
                lpips_batch = lpips_model([lpips_img, lpips_predicted])
                lpips_rec_batch = lpips_model([lpips_img, lpips_reconstructed])

                lpips_batch = tf.reduce_mean(lpips_batch[~np.isnan(lpips_batch)])
                lpips_avg(lpips_batch)

                lpips_rec_batch = tf.reduce_mean(lpips_rec_batch[~np.isnan(lpips_rec_batch)])
                lpips_recon(lpips_rec_batch)
        base_acts = tf.concat(base_acts, axis=0)
        predicted_acts = tf.concat(predicted_acts, axis=0)
        reconstructed_acts = tf.concat(reconstructed_acts, axis=0)

        base_moments = FID.calc_mu_sigma(base_acts)
        predicted_moments = FID.calc_mu_sigma(predicted_acts)
        reconstructed_moments = FID.calc_mu_sigma(reconstructed_acts)

        fid = FID.calc_fid_activations(base_moments, predicted_moments, calc_mu_sigma=False)
        fid_recon = FID.calc_fid_activations(base_moments, reconstructed_moments, calc_mu_sigma=False)
        print(f"\n\nFID for inverse with {n_cycles} cycles is: {fid}")
        print(f"FID for reconstructed with {n_cycles} cycles is: {fid_recon}")
        if test_lpips:
            print(f"LPIPS Score for inverse with {n_cycles} cycles is: {lpips_avg.result()}")
            print(f"LPIPS Score for reconstructed with {n_cycles} cycles is: {lpips_recon.result()}")
            lpips_avg.reset_states()
            lpips_recon.reset_states()
        print(f"Label Prediction Accuracy for inverse with {n_cycles} cycles is: {label_accuracy.result()}")
        print(f"Labal Preciction Accuracy for reconstructed with {n_cycles} cycles is: {recon_accuracy.result()}")
        label_accuracy.reset_states()
        recon_accuracy.reset_states()

    get_fids(test_dataset, n_cycles=0, test_lpips=use_lpips, save_imgs=(True, True), save_to=os.path.join(model_folder, 'test_imgs/'), trailing_zeros=5)
    get_fids(test_dataset, n_cycles=1, test_lpips=use_lpips, save_imgs=(False, True), save_to=os.path.join(model_folder, 'test_imgs/'), trailing_zeros=5)
    get_fids(test_dataset, n_cycles=2, test_lpips=use_lpips, save_imgs=(False, True), save_to=os.path.join(model_folder, 'test_imgs/'), trailing_zeros=5)
    get_fids(test_dataset, n_cycles=3, test_lpips=use_lpips, save_imgs=(False, True), save_to=os.path.join(model_folder, 'test_imgs/'), trailing_zeros=5)

    model.save_models()

if __name__ =='__main__':
    main()


import csv
from os import path
import logging

class FileLoader:

    def __init__(self, paramDict):
        self.paramDict=paramDict

    def __str__(self):
        return 'FileLoader->{}'.format(self.paramFile)

    def __getColumns(self):
        '''Returns the list of columns'''

        if path.exists(self.paramDict['filename']):
            with open(self.paramDict['filename'],mode="r") as paramVals:
                reader = csv.reader(paramVals,delimiter=self.paramDict['delimiter'])
                headers = next(reader, None)
                return headers
        else:
            logging.error("Data file {} does not exists".format(self.paramDict['filename']))
            return None

    def ddlCheck(self):
        ''' Return true if the columns defined in the parameter file exists in the data file'''

        fileColumns=self.__getColumns()
        paramColumns= [v for v in self.paramDict['outColumns']]
        if not self.paramDict['keyColumn'] in paramColumns: paramColumns.append(self.paramDict['keyColumn'])
        # If no headers exits then return false
        if fileColumns==None:
            # logging.error("Data file {} is empty".format(self.paramDict['filename']))
            return False
        # Checking if all the parameter columns exists in the file or not
        elif all(x in fileColumns for x in paramColumns):
            logging.info("Syntax check for {} passed sucessfully".format(self.paramDict['filename']))
            return True
        else:
            logging.error("DDL Check failed : Mismatch between parameter columns and the columns present in the file '{}'".format(self.paramDict['filename']))
            return False

    def getData(self):
        '''Extracts and returns the filtered data'''

        paramDict=self.paramDict
        mainDict=dict()
        if self.ddlCheck():
            with open(paramDict['filename'], mode="r", newline='' ) as psvfile:
                rawData = csv.DictReader(psvfile,delimiter='|')
                for row in rawData:
                    # Filter rows on basis of filterValue passed in the parameter file
                    if not row[paramDict['keyColumn']]==paramDict['filterValue'] : continue
                    # Create the subset dictionary with the required columns for filtered data
                    mainDict[row[paramDict['keyColumn']]]={key : row[key] for key in paramDict['outColumns']}
            if mainDict=={}:
                logging.warning("No data available for filter value : {}".format(paramDict['filterValue']))
            else:
                logging.info("Data ready to be consumed")
            return mainDict
        else:
            logging.error("No data extracted from file : {}".format(self.paramDict['filename']))
            return None