import requests
import shutil
import tempfile
import subprocess
import os
import re

class VMProfTest(object):
    def vmprof_exec(self, *args, cwd=None, jitlog=False, web=False, output=None):
        script = args[0]
        params = []
        if web:
            params += ['--web', '--web-url', self.vmprof_url]
        if jitlog:
            params += ['--jitlog']
        if output:
            params += ['-o', output]
        return self.shell_exec(self.interp, "testvmprof/test/examples/boot_env.py" ,
                               self.tmp + "/" + self.interpname + "-env/bin/activate_this.py",
                               "vmprof", "--", *params, *args, cwd=cwd)

    def shell_exec(self, *args, cwd=None, env={}):
        if cwd == None:
            cwd = os.getcwd()
        print("shexe> %s (pwd: %s)" % (' '.join(args), os.getcwd()))
        proc = subprocess.Popen(args, cwd=cwd,
                                stderr=subprocess.PIPE,
                                stdout=subprocess.PIPE,
                                env=env)
        try:
            outs, err = proc.communicate(timeout=60)
        except subprocess.TimeoutExpired:
            proc.kill()
            raise AssertionError("command: %s. failed %d" % \
                    (' '.join(args), ro.returncode))
        assert proc.returncode == 0, \
            "err\n%s\n\nout\n%s\n" % (err.decode(), outs.decode())
        return outs, err, proc.returncode

class VMProfCPythonTest(VMProfTest):
    def setup_class(self):
        tmp, cpython = setup_local_cpython()
        self.tmp = tmp
        self.interp = os.path.join(tmp, cpython)
        self.interpname = 'cpython'
        self.vmprofargs = "--web --web-url %s" % self.vmprof_url

class VMProfPyPyTest(VMProfTest):
    def setup_class(self):
        tmp, pypy = setup_local_pypy()
        self.tmp = tmp
        self.interp = os.path.join(tmp, pypy)
        self.interpname = 'pypy'
        self.vmprofargs = "--web --web-url %s" % self.vmprof_url

    def jitlog_exec(self, *args, cwd=None, web=False, upload=None, output=None, env={}):
        script = args[0]
        params = []
        if web:
            params += ['--web', '--web-url', self.vmprof_url]
        if output:
            params += ['-o', output]
        if upload:
            params += ['--upload', '--web-url', self.vmprof_url]
        return self.shell_exec(self.interp, "testvmprof/test/examples/boot_env.py" ,
                               self.tmp + "/" + self.interpname + "-env/bin/activate_this.py",
                               "jitlog", "--", *params, *args, cwd=cwd, env=env)

def setup_local_cpython():
    tmp = tempfile.mkdtemp()
    absexe = "/usr/bin/python"
    subprocess.run(["virtualenv", "-p", absexe, "cpython-env"], cwd=tmp)
    subprocess.run(["cpython-env/bin/pip", "install", "--no-cache-dir", "--pre", "vmprof"], cwd=tmp)
    return tmp, os.path.join("cpython-env","bin","python")

def setup_local_pypy(branch='trunk', version='latest', dist='linux64'):
    # uff, hardcoded paths & only for linux
    filename = "pypy.tar.bz2"
    tmp = tempfile.mkdtemp()
    if "TEST_PYPY_EXEC" not in os.environ:
        if "TEST_PYPY_DOWNLOADED" in os.environ:
            shutil.copy(os.environ["TEST_PYPY_DOWNLOADED"], os.path.join(tmp, filename))
        else:
            download_pypy(os.path.join(tmp, filename), branch, version, dist)
        subprocess.run(["tar", "xf", os.path.join(tmp, filename)], cwd=tmp)
        executable = None
        found = False
        for root, dirs, files in os.walk(tmp):
            for dir in dirs:
                if dir.startswith("pypy-"):
                    executable = os.path.join(dir, "bin", "pypy")
                    found = True
                    break
        if not found:
            raise AssertionError("could not setup local pypy")
        absexe = os.path.join(tmp, executable)
    else:
        absexe = executable = os.environ["TEST_PYPY_EXEC"]
    # poor man's dependency resolution
    subprocess.run(["virtualenv", "-p", absexe, "pypy-env"], cwd=tmp)
    subprocess.run(["pypy-env/bin/pip", "install", "--no-cache-dir",
                    "git+https://github.com/vmprof/vmprof-python.git"], cwd=tmp)
    return tmp, os.path.join("pypy-env","bin","python")

def download_pypy(path, branch, version, dist):
    link = "http://buildbot.pypy.org/nightly/{}/pypy-c-jit-{}-{}.tar.bz2".format(branch, version, dist)
    print("downloading", link)
    r = requests.get(link, stream=True)
    if r.status_code == 200:
        with open(path, 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)
            return

    raise Exception("cannot download pypy!")

def output_extract_urls(output):
    found = {}
    last_type = None
    typematch = re.compile(" => Uploading the ([a-z]+) ")
    for line in output.splitlines():
        match = typematch.search(line)
        if match:
            last_type = match.group(1)
            continue
        if last_type is not None:
            found[last_type] = line.strip()
            last_type = None
    return found


#
# @lc app=leetcode.cn id=575 lang=python3
#
# [575] 分糖果
#

from typing import List

# @lc code=start
class Solution:
    def distributeCandies(self, candyType: List[int]) -> int:
        candySet = set(candyType)
        return min(len(candyType) // 2, len(candySet))
# @lc code=end


import os
import sys
from pathlib import Path
from typing import Dict, Callable

import numpy as np
import torch

ROOT_PROJECT = str(Path(os.path.realpath(__file__)).parent.parent.parent)
sys.path[0] = ROOT_PROJECT

def input_constraint_maker(ind: int) -> Callable[[Dict], bool]:
    def f(x: Dict) -> bool:
        return x[f"var_{ind}"] < 0

    return f

def test_rs_acq_opt_in_bobuilder() -> int:
    acq_opt_id = "rs"
    bo_builder = BoBuilder(model_id="gp_to", acq_opt_id=acq_opt_id, acq_func_id="ei", tr_id="basic")
    optimizer = bo_builder.build_bo(
        search_space=search_space, n_init=10, input_constraints=input_constraints, dtype=dtype, device=device
    )
    print(optimizer.name, task.name)
    for i in range(20):
        x_next = optimizer.suggest(3)
        y_next = task(x_next)
        optimizer.observe(x_next, y_next)
        print(f'Iteration {i + 1:>4d} - f(x) - {y_next[0][0]:.3f} - f(x*) - {optimizer.best_y:.3f}')

    return 0

def test_rs_acq_opt() -> int:
    x_train_pd = search_space.sample(100)
    x_train = search_space.transform(x_train_pd)
    y_train = torch.tensor(task(x_train_pd))

    kernel = TransformedOverlap()
    model = ExactGPModel(search_space, 1, kernel, dtype=dtype, device=device)
    model.fit(x_train, y_train)

    acq_opt = RandomSearchAcqOptimizer(search_space=search_space, input_constraints=input_constraints)

    x = search_space.transform(search_space.sample(1))
    acq_func = acq_factory("lcb")
    new_points = acq_opt.optimize(
        x=x,
        n_suggestions=3,
        x_observed=x_train,
        model=model,
        acq_func=acq_func,
        acq_evaluate_kwargs={},
        tr_manager=None
    )
    new_points = search_space.inverse_transform(new_points)
    assert np.all(
        [[input_constraints[i](new_points.iloc[[j]]) for i in range(len(input_constraints))] for j in
         range(len(new_points))])
    print(new_points)
    return 0

if __name__ == '__main__':
    from mcbo.task_factory import task_factory
    from mcbo.optimizers import BoBuilder
    from mcbo.acq_funcs import acq_factory
    from mcbo.acq_optimizers.random_search_acq_optimizer import RandomSearchAcqOptimizer
    from mcbo.models import ExactGPModel
    from mcbo.models.gp.kernels import TransformedOverlap

    dtype = torch.float64
    device = torch.device("cuda")
    task = task_factory('levy', num_dims=[2, 3, 2], variable_type=['int', 'nominal', 'num'],
                        num_categories=[None, 21, None])
    search_space = task.get_search_space(dtype=dtype)

    input_constraints = [input_constraint_maker(i) for i in range(1, 4)]

    test_rs_acq_opt()


import pymongo
from bson.code import Code

myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["arth34"]
mycol = mydb["orders"]

mylist = [
{ "_id": 1, "cust_id": "Ant O. Knee",  "price": 25, "items": [ { "sku": "oranges", "qty": 5, "price": 2.5 }, { "sku": "apples", "qty": 5, "price": 2.5 } ], "status": "A" },
{ "_id": 2, "cust_id": "Ant O. Knee",  "price": 70, "items": [ { "sku": "oranges", "qty": 8, "price": 2.5 }, { "sku": "chocolates", "qty": 5, "price": 10 } ], "status": "A" },
{ "_id": 3, "cust_id": "Busby Bee",  "price": 50, "items": [ { "sku": "oranges", "qty": 10, "price": 2.5 }, { "sku": "pears", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 4, "cust_id": "Busby Bee",  "price": 25, "items": [ { "sku": "oranges", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 5, "cust_id": "Busby Bee",  "price": 50, "items": [ { "sku": "chocolates", "qty": 5, "price": 10 } ], "status": "A"},
{ "_id": 6, "cust_id": "Cam Elot",  "price": 35, "items": [ { "sku": "carrots", "qty": 10, "price": 1.0 }, { "sku": "apples", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 7, "cust_id": "Cam Elot",  "price": 25, "items": [ { "sku": "oranges", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 8, "cust_id": "Don Quis",  "price": 75, "items": [ { "sku": "chocolates", "qty": 5, "price": 10 }, { "sku": "apples", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 9, "cust_id": "Don Quis",  "price": 55, "items": [ { "sku": "carrots", "qty": 5, "price": 1.0 }, { "sku": "apples", "qty": 10, "price": 2.5 }, { "sku": "oranges", "qty": 10, "price": 2.5 } ], "status": "A" },
{ "_id": 10, "cust_id": "Don Quis",  "price": 25, "items": [ { "sku": "oranges", "qty": 10, "price": 2.5 } ], "status": "A" }
]

x = mycol.insert_many(mylist)

for x in mycol.find():
	print(x)

mapFunction1 = Code("function() { emit(this.cust_id, this.price);}")
reduceFunction1 = Code("function(keyCustId, valuesPrices) {return Array.sum(valuesPrices)}")
result = mycol.map_reduce(mapFunction1, reduceFunction1, "result")

rescol = mydb["result"]
for y in rescol.find():
	print(y)


__author__ = 'jagzviruz'

import re
import os
import errno

from bs4 import BeautifulSoup
import requests
import sys, threading, time
import pycurl

import pymongo
from pymongo import MongoClient

client = MongoClient()
db = client.medplus_data

# Step 1 : Open the category listing page, and download the category pages, to extract the Product IDs

class DownLoadPageWithPyCurlInThread(threading.Thread):
    def __init__(self, url, output_file):
        threading.Thread.__init__(self)
        self.curl = pycurl.Curl()
        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.WRITEDATA, output_file)
        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
        self.curl.setopt(pycurl.MAXREDIRS, 5)
        self.curl.setopt(pycurl.NOSIGNAL, 1)

    def run(self):
        self.curl.perform()
        self.curl.close()
        sys.stdout.write(". ")
        sys.stdout.flush()

class GetLinksFromFile:
    def __init__(self, start_file):
        text_file = open(start_file, "r")
        self.start_urls = text_file.readlines()
        text_file.close()

def processDownloadedCategoryFile(filepath):
    print("\r\n** Processing "+filepath + " with BeautifulSoup\n\r")
    file_handle = open(filepath,encoding='ISO-8859-1')
    content = file_handle.read()
    beautiful_soup = BeautifulSoup(content, "lxml")

    category_name = beautiful_soup.find('h1')

    if category_name:
        category_name = category_name.string.strip()
    else:
        category_name = ''

    print("\r\n*** Category Name : "+ category_name)
    ind = filepath.rfind('/')
    folder_path = filepath[0:ind + 1]
    cat_modified_name = re.sub(r'\W+', '',category_name)
    new_file_path = folder_path + cat_modified_name + ".html";
    print("\r\n**** Renaming " + filepath + ' to ' + new_file_path)
    os.rename(filepath, new_file_path)

    prodLinksFile = open('prodlinks.txt', 'a')

    obj = {
        'category_name': str(category_name),
        'category_prod_links': []
    }

    replace_pattern = re.compile(r'[\[\]\"]')
    inputStringWithAllProductIds = beautiful_soup.find('input', {'id': 'productIdString'})

    if inputStringWithAllProductIds is None:
        categoriesWithNoProducts = open('CategoriesWithNoProducts.txt', 'a')
        categoriesWithNoProducts.write(category_name + '\r\n')
        categoriesWithNoProducts.close()
        return

    inputStringWithAllProductIds = inputStringWithAllProductIds.get('value')
    inputStringWithAllProductIds = replace_pattern.sub('', inputStringWithAllProductIds)
    inputStringWithAllProductIds = inputStringWithAllProductIds.replace('\u0026', '&')
    inputStringWithAllProductIds = inputStringWithAllProductIds.split(",")

    prodPath = 'http://www.medplusmart.com/product/'+ cat_modified_name +'/'

    for prod_id in inputStringWithAllProductIds:
        small_obj = {
            "prod_id": prod_id,
            "prod_cat": category_name,
            "prod_link": prodPath + prod_id
        }
        prod_path = prodPath + prod_id

        item = checkIfProductPresentInProducts(prod_id)
        if item is None:
            link_obj = {
                "prod_id": prod_id,
                "prod_cat": [category_name],
                "prod_link": prod_path
            }

            storeDataInCollection('products', link_obj)
            prodLinksFile.write(prod_id + ':' + prod_path + "\r\n")

        else:
            item['prod_cat'].append(category_name)
            storeDataInCollection('products', item, True)
        obj['category_prod_links'].append(small_obj)

    storeDataInCollection('categories', obj)
    prodLinksFile.close()

def checkIfProductPresentInProducts(prod_id):
    prod = db.products.find_one({
        'prod_id': prod_id
    })

    if prod:
        return prod
    else:
        return None

def storeDataInCollection(collection_name, doc_to_store, updateMode=False):

    if updateMode:
        item = db[collection_name].find_one({
         '_id' : doc_to_store['_id']
        })

        item.update(doc_to_store)

        db[collection_name].update_one({
            '_id': item['_id']
        }, {
            '$set': item
        }, upsert=False)

    else:
        db[collection_name].insert(doc_to_store)

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i+n]

def download (urls, output_folder):
    fileno = 0
    t1 = time.time()
    for url in urls:
        f = open(output_folder + str(fileno) + ".html", "wb")
        t = DownLoadPageWithPyCurlInThread(url.rstrip(), f)
        t.start()
        fileno = fileno + 1
        t.join()
        f.close()
    t2 = time.time()
    print( "\n** %d seconds elapsed for %d uris" % (int(t2-t1), len(urls)))

def parallel_download (urls, output_folder):
    threads = []
    fileno = 0

    # Start one thread per URI in parallel
    t1 = time.time()
    for url in urls:
        f = open(output_folder + str(fileno) + ".html", "wb")
        t = DownLoadPageWithPyCurlInThread(url.rstrip(), f)
        t.start()
        threads.append((t, f))
        fileno = fileno + 1

    # # Wait for all threads to finish
    for thread, file in threads:
        thread.join()
        file.close()
        processDownloadedCategoryFile( file.name)

    t2 = time.time()
    print("\n** Multithreading, %d seconds elapsed for %d uris" % (int(t2-t1), len(urls) ) )

def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def startDownloadingCategoryPagesInParallelThreads(category_urls, path='somedir/', num_threads=1):

    make_sure_path_exists(path)

    if (num_threads > 30):
        print ("*** It is not advisable to run more than 30 threads in parallel.\r\n")
        exit()

    elif (num_threads == 1):
        download(category_urls, path)

    else:
        subs = chunks(category_urls, num_threads)
        for sub in subs:
            parallel_download(sub, path)

# Step 1 : Hit the Master category page and build the mongo collections that will store the data.

# startCrawl = GetLinksFromFile("start-urls.txt")
# ind = 0
# print("* Crawler Started")
# print("*** Found " + str(len(startCrawl.start_urls)) + " urls to start crawling from")

# for start_url in startCrawl.start_urls:
#     rootpath = str(start_url)[0:str(start_url).index('/pharmaHome')]
#     ind += 1
#     r = requests.get(start_url)
#     beautiful_soup = BeautifulSoup(r.content, "lxml")
#     divs = beautiful_soup.find_all('ul', {'class': 'brandlist-inline'})
#     links = beautiful_soup.find_all('a', {'href': re.compile('^/drugsCategory/*')})

#     print("***** Found "+str(len(links)) + " categories *** \r\n")
#     print("******** Storing all category links in one file : 'AllCategoryLinks.txt' ,\r\n")
#     print("********  which will be used to crawl to get product details")

#     fileToStoreLinksIn = open('AllCategoryLinks.txt', 'w')

#     for link in links:
#         w = rootpath
#         w = w + str(link.get('href')) + '\r\n'
#         fileToStoreLinksIn.write(w)
#     fileToStoreLinksIn.close()

# print("* Crawler will start crawling links in AllCategoryLinks.txt")

# startCrawl = GetLinksFromFile("AllCategoryLinks.txt")

# startDownloadingCategoryPagesInParallelThreads(startCrawl.start_urls, 'categories/',  30)

# Step 2 :
source_filename=    "prodlinks.txt"
completed_filename= "completed_scraping.txt"
error_filename=     "error_scraping.txt"

def fetchAllProdlinks() :
    file = open(source_filename,"r")
    links = file.readlines()
    file.close()

    return links

def updateProductMetaInMongo(prod_id, obj):
    item = db.products.find_one({
        'prod_id': prod_id
    })
    item.update(obj)
    db.products.update_one({
        '_id': item['_id']
    }, {
        '$set': obj
    },upsert=False)

    print("Updated : "+ prod_id)

def getPageContentWithRequests(url, pid):
    try:
        r = requests.get(url)
        beautiful_soup = BeautifulSoup(r.content, "lxml")
        prod_name = str(beautiful_soup.find('h1').text)

        dosage_form = beautiful_soup.find('strong',{ 'itemprop':"dosageForm" } )
        if not dosage_form is None:
            dosage_form = dosage_form.text.strip()
        else:
            dosage_form = 'NA'

        comp = beautiful_soup.find('strong',{ 'itemprop':"activeIngredient" } )
        if not comp is None:
            comp = comp.text.strip()
        else:
            comp = 'NA'

        pack_size = beautiful_soup.find('strong',{ 'itemprop':"activeIngredient" } )
        if not pack_size is None:
            pack_size = pack_size.text.strip()
        else:
            pack_size = 'NA'

        manufacturer = beautiful_soup.find('strong',{ 'itemprop':"name" } )
        if not manufacturer is None:
            manufacturer = manufacturer.text.strip()
        else:
            manufacturer = 'NA'

        mrp = beautiful_soup.find('span',{ 'itemprop':"costPerUnit" } )
        if not mrp is None:
            mrp = mrp.text.strip()
        else:
            mrp = 'NA'

        prod_meta = {
            'comp':  comp,
            'pack_size': pack_size,
            'manufacturer': manufacturer,
            'mrp': mrp,
            'form': dosage_form
        }

        alternates = beautiful_soup.find_all('tr',{'itemprop' : 'relatedDrug'})

        alternatives = []

        for alternate in alternates:
            alternate = BeautifulSoup(str(alternate), "lxml")
            link = alternate.find('a').get('href')
            prod_id = link.split('/')[-1]

            product_name = alternate.find('h5')
            if not product_name is None:
                product_name = product_name.text.strip()
            else:
                product_name = 'NA'

            tmp = {
                'prod_name' : product_name,
                'prod_id' : prod_id,
                'prod_link' : "http://www.medplusmart.com" + link,
                'prod_mfg' : alternate.find('small', {'itemprop' : 'manufacturer'}).find('span').text.strip()
            }
            alternatives.append(tmp)

        prod_information_blocks = beautiful_soup.find_all('div', {'class': 'tabbable'})

        prod_information = []

        for block in prod_information_blocks:
            cont = BeautifulSoup(str(block), 'lxml')
            cont = cont.find('div', {'class': 'container-fluid'})

            if cont is None:
                continue

            titles = cont.find_all('h4')
            descs = cont.find_all('p')
            index = 0

            for title in titles:
                desc = descs[index]

                if not desc is None:
                    desc = desc.text.strip()
                else:
                    desc = 'NA'

                tmp = {
                    'title' : title.text.strip(),
                    'description' : desc
                }

                prod_information.append(tmp)
                index = index + 1

        obj = {
            'prod_name' : prod_name,
            'prod_meta' : prod_meta,
            'alternatives' : alternatives,
            'prod_information' : prod_information
        }

        updateProductMetaInMongo(pid, obj)

    except Exception:
        error_file = open(error_filename, 'a')
        error_file.write(pid + ':' + url + '\r')
        error_file.close()
        pass

# ----------------------------------------------------------------------------------------------------------------------

prod_links_lines = fetchAllProdlinks()
lines_written = 0

for line in prod_links_lines:
    if lines_written == 0 :
        completed_file = open(completed_filename, "a")

    parts = line.split(':', 1)
    prod_id = parts[0]
    prod_link = parts[1]
    getPageContentWithRequests(prod_link, prod_id)
    completed_file.write(line + '\r')
    lines_written = lines_written + 1
    if lines_written == 100:
        completed_file.close()
        lines_written = 0


from time import sleep
import sys

import gym

map_size = sys.argv[1]

if map_size == '4':
    map_name = '4x4'
elif map_size == '8':
    map_name = '8x8'

env = gym.make('FrozenLake-v1', desc=None, map_name=map_name, is_slippery=True)

observation = env.reset()
for _ in range(100):
    sleep(0.2)
    render = env.render(mode='human')
    action = env.action_space.sample()  # uniformly random actions
    observation, reward, done, info = env.step(action)

    if done:
        observation = env.reset()
env.close()


import logging

import attr

from subiquity.common.apidef import API
from subiquity.common.types import WSLConfigurationBase
from subiquity.server.controller import SubiquityController
from subiquitycore.context import with_context
from system_setup.common.wsl_conf import default_loader
from system_setup.common.wsl_utils import convert_if_bool

log = logging.getLogger("system_setup.server.controllers.wslconfbase")

class WSLConfigurationBaseController(SubiquityController):
    endpoint = API.wslconfbase

    autoinstall_key = model_name = "wslconfbase"
    autoinstall_schema = {
        "type": "object",
        "properties": {
            "automount_root": {"type": "string"},
            "automount_options": {"type": "string"},
            "network_generatehosts": {"type": "boolean"},
            "network_generateresolvconf": {"type": "boolean"},
        },
        "additionalProperties": False,
    }

    def __init__(self, app):
        super().__init__(app)

        # load the config file
        root_dir = self.app.base_model.root
        data = default_loader(root_dir)

        if data:
            proc_data = {key: convert_if_bool(value) for (key, value) in data.items()}
            conf_data = WSLConfigurationBase(**proc_data)
            self.model.apply_settings(conf_data)

    def load_autoinstall_data(self, data):
        if data is not None:
            identity_data = WSLConfigurationBase(**data)
            self.model.apply_settings(identity_data)

    @with_context()
    async def apply_autoinstall_config(self, context=None):
        pass

    def make_autoinstall(self):
        r = attr.asdict(self.model.wslconfbase)
        return r

    async def GET(self) -> WSLConfigurationBase:
        data = WSLConfigurationBase()
        if self.model.wslconfbase is not None:
            data.automount_root = self.model.wslconfbase.automount_root
            data.automount_options = self.model.wslconfbase.automount_options
            data.network_generatehosts = self.model.wslconfbase.network_generatehosts
            data.network_generateresolvconf = (
                self.model.wslconfbase.network_generateresolvconf
            )
        return data

    async def POST(self, data: WSLConfigurationBase):
        self.model.apply_settings(data)
        await self.configured()


# -*- coding: utf-8 -*-
import sys
import json
import bisect
import hashlib
import logging
sys.path.append("../../")

from hashring.common import exceptions as exc
from hashring.common.config import Const

logger = logging.getLogger(__name__)
logger.setLevel(level=logging.INFO)
handler = logging.FileHandler(Const.LOG_FILE_NAME_RUN)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s -%(lineno)d- %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

def get_hash(raw_str):
    """将字符串映射到2^32的数字中"""

    md5_str = hashlib.md5(raw_str.encode('utf-8')).hexdigest()
    return int(md5_str, 16) % (Const.MOD_NUM)

class DataManager(object):
    """Data manager."""

    def __init__(self, path):
        # DataManager.filename=path
        self.dev_inf = None  # 设备信息
        self.filename = path

    def load(self):
        u"""加载文件.
        :param filename: 文件名
        """

        with open(self.filename) as f:
            self.dev_inf = json.load(f)

    def save(self):
        u"""保存文件.
        :param filename: 文件名
        """

        with open(self.filename, "w") as f:
            json.dump(self.dev_inf, f)

class RingBuilder(object):
    """ring builder."""

    def __init__(self, data_manager):
        self.sorted_cache_list = []  # 排序好的hash值(虚拟节点)
        self.cache_node = dict()  # 存储虚拟节点到物理设备的hash映射
        self.data_manager = data_manager
        logger.info("init device")
        for dev_id_key, item in self.data_manager.dev_inf.items():
            part_num = item[Const.PART_NUM]
            dev_weight = item[Const.DEV_WEIGHT]
            self.rebalance(dev_id_key, Const.ADD, 0, part_num * dev_weight)
        logger.info("init device success!")

    def add_dev(self, dev_id, dev_name=None, dev_weight=None, part_num=None):
        u"""添加设备.
        :param dev_id: 待添加的设备信息
        """

        logger.info("Add device")
        # params type check
        if dev_id in self.data_manager.dev_inf.keys():
            return
        if dev_id and not isinstance(dev_id, str):
            raise TypeError("dev_id must be str type")
        if dev_name and not isinstance(dev_name, str):
            raise TypeError("dev_name must be str type")
        if dev_weight and  not isinstance(dev_weight, int):
            raise TypeError("dev_weight must be int type")
        if part_num and not isinstance(part_num, int):
            raise TypeError("part_num must be int type")

        # 更新设备信息
        added_dev = {dev_id: {Const.DEV_NAME:dev_name, Const.DEV_WEIGHT:dev_weight, Const.PART_NUM:part_num}}
        self.data_manager.dev_inf.update(added_dev)

        # rebalance
        virtual_node_num = dev_weight * part_num
        self.rebalance(dev_id,
            add_or_remove=Const.ADD,
            start_node_index=0,
            end_node_index=virtual_node_num)

        # save dev_inf
        self.data_manager.save()
        logger.info("Add device success!")
        return 0

    def update_dev(self, dev_id, dev_weight=None):
        u"""更新设备信息.

        :param dev_id: 设备ID
        :param weight: 权重
        """

        logger.info("Update device")
        if dev_id not in self.data_manager.dev_inf:
            return
        if dev_id and not isinstance(dev_id, str):
            raise TypeError("dev_id must be str type")
        if dev_weight and not isinstance(dev_weight, int):
            raise TypeError("dev_weight must be int type")

        # rebalance
        info_of_dev_id = self.data_manager.dev_inf[dev_id]  # dev_id 的设备信息， 字典类型
        virtual_num = info_of_dev_id[Const.PART_NUM]
        old_dev_weight = info_of_dev_id[Const.DEV_WEIGHT]
        old_virtual_node_num = old_dev_weight * virtual_num
        new_virtual_node_num = dev_weight * virtual_num
        logger.info("new_virtual_node_num:{}".format(new_virtual_node_num))
        logger.info("old_virtual_node_num:{}".format(old_virtual_node_num))
        if old_virtual_node_num < new_virtual_node_num:
            self.rebalance(dev_id,
                add_or_remove=Const.ADD,
                start_node_index = old_virtual_node_num,
                end_node_index = new_virtual_node_num)
        elif old_virtual_node_num > new_virtual_node_num:
            self.rebalance(dev_id,
                add_or_remove = Const.REMOVE,
                start_node_index = new_virtual_node_num,
                end_node_index = old_virtual_node_num)
        # save dev_inf
        self.data_manager.dev_inf[dev_id].update({Const.DEV_WEIGHT: dev_weight})
        self.data_manager.save()
        logger.info("Update device success! dev_id = {}, dev_weight = {}".format(dev_id, dev_weight))
        return 0

    def remove_dev(self, dev_id):
        u"""删除设备.

        :param dev_id: 待删除的设备ID
        """

        logger.info("Remove device")
        item = self.data_manager.dev_inf[dev_id]  # dev_id : str
        self.virtual_num = item[Const.PART_NUM]
        del self.data_manager.dev_inf[dev_id]
        virtual_node_num = self.virtual_num * item[Const.DEV_WEIGHT]

        self.rebalance(dev_id,
            add_or_remove=Const.REMOVE,
            start_node_index=0,
            end_node_index=virtual_node_num)
        self.data_manager.save()
        logger.info("Remove device success ! dev_id = {}".format(dev_id))
        return 0

    # @TODO params check
    def rebalance(self, dev_id, add_or_remove, start_node_index, end_node_index):
        u"""重新平衡ring."""

        logger.info("Rebalance device!")
        logger.info("Rebalance: dev_id = {}, add_or_remove = {}".format(dev_id, add_or_remove))
        logger.info("Rebalance: start_node_index ={}, end_node_index = {}".format(start_node_index,
                                                                            end_node_index))
        for index in range(start_node_index, end_node_index):
            node_hash = get_hash("%s_%s" % (dev_id, index))
            if add_or_remove == Const.ADD:
                bisect.insort(self.sorted_cache_list, node_hash)
                self.cache_node[node_hash] = dev_id
            else:
                self.sorted_cache_list.remove(node_hash)
                del self.cache_node[node_hash]
        logger.info("Rebalance device success!")
        return 0

    def hash_dev(self, key):
        u"""获取指定key hash到的设备."""
        node_index = bisect.bisect_left(self.sorted_cache_list, key) #
        node_index = node_index % len(self.sorted_cache_list)  # 若比最大的node hash还大，分发给第一个node
        return self.cache_node[self.sorted_cache_list[node_index]]

    # 保存文件到本地
    def save_hash_to_node(self, filename1, filename2):
        with open(filename1,"w")as f:
            json.dump(self.sorted_cache_list, f)  # 保存哈希列表到本地
        with open(filename2, "w")as f:
            json.dump(self.cache_node, f)  # 保存字典（"哈希值-设备ID"）到本地
        return 0


from collections import deque
import threading
import randomgen
import numpy as np

from utils.sampling.perlin import create_perlin_noise, calc_fade

class SamplingProvider:
    """
    Multi-threaded util that runs in the background and precalculates noise samples. To be used in with statement!
    Even with the GIL, this is helpful because the GIL is released while we're waiting for model predictions.
    - If we're attacking a local model, TensorFlow releases GIL while blocking on sess.run
    - If we're attacking a remote model, blocking calls like HTTPRequest also release GIL
    - The RNG we use also releases GIL

    We can use that time to fill the queues with the next batch of samples.

    This implementation may be a bit over-engineered. The background is that we used to run this on a AWS instance that would sometimes
    crash if running >2 threads. For that reason, we wrote this util so we can fill (and block on) multiple deques with a single thread.
    """
    def __init__(self, shape, n_threads=1, queue_lengths=40):
        self.shape = shape
        self.n_threads = n_threads
        self.queue_lengths = queue_lengths
        self.queue_normal = deque()
        self.queue_perlin = deque()

        self.perlin_fade = calc_fade(shape[0], freq=20)             # Tiny ImageNet: 5 works well. ImageNet: 20 is better.
        self.perlin_color = True

        # Manually reimplemented Queue locking to cover 2 deques instead of 1.
        self.lock = threading.Lock()
        self.cv_not_full = threading.Condition(self.lock)           # Producer condition
        self.cv_not_empty = threading.Condition(self.lock)          # Consumer condition
        self.is_running = True
        self.threads = []
        for thread_id in range(n_threads):
            thread = threading.Thread(target=self._thread_fun, args=(thread_id,))
            thread.start()
            self.threads.append(thread)

    def __enter__(self):
        # Threads already started at __init__
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        # Stop all the threads.
        for thread in self.threads:
            thread.do_run = False

        self.is_running = False

        # Stop producers.
        with self.cv_not_full:
            self.cv_not_full.notify_all()

        # Stop consumers, if any.
        # Usually, Consumers are from the same thread that creates and __exit__s this object. So there will be none waiting at this point.
        # If, however, there is a consumer from a background thread, they could be still waiting, and then they will receive an
        #  InterruptedError here.
        with self.cv_not_empty:
            self.cv_not_empty.notify_all()

        for thread in self.threads:
            thread.join()
        print("SamplingProvider: all threads stopped.")

    def _thread_fun(self, thread_id):
        # create a thread-specifc RNG
        rng = randomgen.RandomGenerator(randomgen.Xoroshiro128(seed=20 + thread_id))
        rnd_normal = None
        rnd_perlin = None

        t = threading.currentThread()
        while getattr(t, 'do_run', True):

            # Prepare one of each sampling patterns
            if rnd_normal is None:
                rnd_normal = rng.standard_normal(size=self.shape, dtype='float32')
                rnd_normal /= np.linalg.norm(rnd_normal)
            if rnd_perlin is None:
                rnd_perlin = create_perlin_noise(px=self.shape[0], color=self.perlin_color, batch_size=1,
                                                  normalize=True, precalc_fade=self.perlin_fade)[0]

            # Lock and put them into the queues.
            with self.cv_not_full:
                if len(self.queue_normal) >= self.queue_lengths and len(self.queue_perlin) >= self.queue_lengths:
                    self.cv_not_full.wait()

                # Fill one or both queues.
                if len(self.queue_normal) < self.queue_lengths:
                    self.queue_normal.append(rnd_normal)
                    rnd_normal = None
                if len(self.queue_perlin) < self.queue_lengths:
                    self.queue_perlin.append(rnd_perlin)
                    rnd_perlin = None

                self.cv_not_empty.notify_all()

    def get_normal(self, return_stats=False):
        """
        Returns a std-normal noise vector, normalized to L2=1
        """
        with self.cv_not_empty:
            while len(self.queue_normal) == 0:
                self.cv_not_empty.wait()
                if not self.is_running:
                    raise InterruptedError("Trying to consume an item, but was already shut down!")

            sample = self.queue_normal.popleft()
            self.cv_not_full.notify()

        # Unused in this implementation.
        # Use this field to record statistics of successful samples, see BlackboxWrapper.log_img
        if return_stats:
            return sample, {}
        else:
            return sample

    def get_perlin(self, return_stats=False):
        """
        Returns a perlin noise vector, normalized to L2=1
        """
        with self.cv_not_empty:
            while len(self.queue_perlin) == 0:
                self.cv_not_empty.wait()
                if not self.is_running:
                    raise InterruptedError("Trying to consume an item, but was already shut down!")

            sample = self.queue_perlin.popleft()
            self.cv_not_full.notify()

        # Unused in this implementation.
        # Use this field to record statistics of successful samples, see BlackboxWrapper.log_img
        if return_stats:
            return sample, {}
        else:
            return sample


import json
from googleapiclient.discovery import build, Resource
from google.oauth2.credentials import Credentials

from infrastructure.models import Environment
from common.methods import set_progress
from resourcehandlers.gcp.models import GCPHandler, GCPProject

RESOURCE_IDENTIFIER = 'instance_name'

def discover_resources(**kwargs):
    discovered_google_bigtables = []

    #create a set of all projects
    gcp_envs = Environment.objects.filter(
        resource_handler__resource_technology__name="Google Cloud Platform")
    projects = {env.gcp_project for env in gcp_envs}

    for handler in GCPHandler.objects.all():
        set_progress('Connecting to Google BigTable for \
                      handler: {}'.format(handler))

        #only get bigtables of projects in the set
        for env in Environment.objects.filter(resource_handler_id=handler.id):
            project = env.gcp_project
            if project not in projects:
                continue

            wrapper = create_bigtable_api_wrapper(handler)
            if not wrapper:
                set_progress("Could not connect to Google Cloud.  Skipping this resource handler.")
                continue
            gcp_project_name = GCPProject.objects.get(id=project).gcp_id
            set_progress("Connection to Google Cloud established")
            instances = list_bigtables(wrapper, gcp_project_name).get("instances", None)
            if instances:
                for bigtable in instances:
                    if bigtable.get("displayName"):
                        discovered_google_bigtables.append(
                            {
                                'name': bigtable.get("displayName", "noName"),
                                'instance_name': bigtable.get("displayName", "noName"),
                                'google_rh_id': handler.id,
                                'project_id': gcp_project_name,
                            }
                        )
                #remove project from the set after getting its bigtables
                projects.discard(gcp_project_name)

    return discovered_google_bigtables

def list_bigtables(wrapper: Resource, project_id: str) -> dict:
    """
    Get all bigtable instances in a given project
    https://cloud.google.com/bigtable/docs/reference/admin/rest/v2/projects.instances/list
    """
    proj_str = f"projects/{project_id}"
    inst_list_request = wrapper.projects().instances().list(parent=proj_str)
    return inst_list_request.execute()

def create_bigtable_api_wrapper(gcp_handler: GCPHandler) -> Resource:
    """
    Using googleapiclient.discovery, build the api wrapper for the bigtableadmin api:
    https://googleapis.github.io/google-api-python-client/docs/dyn/bigtableadmin_v2.projects.instances.html
    """
    if not gcp_handler.gcp_api_credentials:
        set_progress("Could not find Google Cloud credentials for this reource handler.")
        return None
    credentials_dict = json.loads(gcp_handler.gcp_api_credentials)
    credentials = Credentials(**credentials_dict)
    bigtable_wrapper: Resource = build("bigtableadmin", "v2", credentials=credentials, cache_discovery=False)
    return bigtable_wrapper


import numpy as np
import nnfs
from nnfs.datasets import spiral_data

from lib.layers.dense import Dense
from lib.layers.activation import Activation
from lib.activation_functions import *
from lib.loss.categorical_crossentropy import CategoricalCrossentropy
nnfs.init()

X, y = nnfs.datasets.spiral_data(samples=100, classes=3)

dense1 = Dense(2, 3)
activation1 = Activation(activation_function=ReLU)

dense2 = Dense(3, 3)
activation2 = Activation(activation_function=softmax)

loss_function = CategoricalCrossentropy()

dense1.forward(X)
activation1.forward(dense1.output)
dense2.forward(activation1.output)
activation2.forward(dense2.output)

print(activation2.output[:5])

# Perform a forward pass through loss function
# it takes the output of second dense layer here and returns loss
loss = loss_function.calculate(activation2.output, y)
# Print loss value
print('loss:', loss)

predictions = np.argmax(activation2.output, axis=1)
if len(y.shape) == 2:
 y = np.argmax(y, axis=1)
accuracy = np.mean(predictions==y)
# Print accuracy
print('acc:', accuracy)


from os import path
from math import log

import torch
from torch import Tensor
from torch.utils.data import Dataset, TensorDataset
from torchvision.datasets import ImageFolder
from torchvision.transforms import Compose, Resize, ToTensor
import numpy as np
import pandas as pd

def _format_celeba_dataset(
    images: Tensor,
    labels: Tensor,
    start_sigma: float = 1.0,
    end_sigma: float = 0.01,
) -> Dataset:
    idx = torch.randperm(images.shape[0])
    images = images[idx]
    labels = labels[idx]

    sigmas = torch.linspace(log(start_sigma), log(end_sigma), idx.shape[0]).exp()
    noise = torch.randn_like(images) * sigmas.view(-1, 1, 1, 1)
    inputs = images + noise
    targets = -noise

    return TensorDataset(inputs, labels, targets)

def load_celeba_data() -> Dataset:
    images = ImageFolder(
        path.join("data", "celeba-dataset", "img_align_celeba"),
        transform=Compose([Resize((48, 64)), ToTensor()])
    )
    images = torch.stack([image[0] for image in images], dim=0)

    labels = pd.read_csv(path.join("data", "celeba-dataset", "list_attr_celeba.csv"))
    labels = torch.from_numpy(labels.to_numpy()[:, 1:].astype(np.float32))

    return _format_celeba_dataset(
        images, labels, start_sigma=1.0, end_sigma=0.01
    )

def load_celeba_small_data() -> Dataset:
    images = ImageFolder(
        path.join("data", "celeba-small-dataset", "img_align_celeba"),
        transform=Compose([Resize((48, 64)), ToTensor()])
    )
    images = torch.stack([image[0] for image in images], dim=0)

    labels = pd.read_csv(path.join("data", "celeba-small-dataset", "list_attr_celeba.csv"))
    labels = torch.from_numpy(labels.to_numpy()[:, 1:].astype(np.float32))

    return _format_celeba_dataset(
        images, labels, start_sigma=1.0, end_sigma=0.01
    )


import os

import kaggle

from dask import dataframe as dd

class TxAnalysis:

    def __init__(self):
        return

    @staticmethod
    def load_dataset(dataset_ref: str):
        dataset_path_parent = '../dataset/'
        dataset_path = os.path.join(dataset_path_parent,
                                    "bank_transactions.csv")
        # Ensure the Kaggle API client is properly configured
        kaggle.api.authenticate()
        # Download the specified dataset
        kaggle.api.dataset_download_files(dataset_ref,
                                          path=dataset_path_parent,
                                          unzip=True)

        # Import a csv dataset from dask
        dataset = dd.read_csv(dataset_path)

        return dataset

    @staticmethod
    def prepare_dataset(dataset):
        dataprep = dataset.groupby(
            "CustomerID")["TransactionAmount (INR)"].agg(['mean', 'count'])
        dataprep = dataprep.reset_index()
        dataprep = dataprep.rename(
            columns={
                "TransactionAmount (INR)": "amount",
                "mean": "avg_amount",
                "count": "nb_tx",
            })
        return dataprep

    @staticmethod
    def analyse(dataprep):
        result = dataprep
        return result

    @staticmethod
    def train_model(dataprep):
        model = None
        return model

    def run(self):
        ref_dataset = "shivamb/bank-customer-segmentation"
        dataset = TxAnalysis.load_dataset(dataset_ref=ref_dataset)
        # print(type(dataset))
        # print(dataset.head(compute=True))
        # print(len(dataset))

        dataprep = TxAnalysis.prepare_dataset(dataset)
        dataprep = dataprep.sort_values(by=['nb_tx'], ascending=False)
        print(dataprep.head(compute=True))

        # analysis = TxAnalysis.analyse(dataprep)
        # model = TxAnalysis.train_model(dataprep)

        return


import pygame
import random

pygame.init()
altura = 300
largura = 800
tamanho = (largura, altura)
pygameDisplay = pygame.display
pygameDisplay.set_caption("SharkAttack")
bg = pygame.image.load("assets/fundo.png.jpeg")
bg_destroy = pygame.image.load("assets/bg-destroy.jpeg.jpeg")

gameDisplay = pygame.display.set_mode(tamanho)
gameEvents = pygame.event
clock = pygame.time.Clock()

gameIcon = pygame.image.load("assets/tubarão.ico")
pygameDisplay.set_icon(gameIcon)

pink = (255, 125, 198)
black = (0, 0, 0)
white = (255, 255, 255)

explosaoSound = pygame.mixer.Sound("assets/explosao.wav.mpeg")
explosaoSound.set_volume(0.5)

def dead(pontos):
    gameDisplay.blit(bg_destroy, (0, 0))
    pygame.mixer.music.stop()
    pygame.mixer.Sound.play(explosaoSound)
    fonte = pygame.font.Font("freesansbold.ttf", 50)
    texto = fonte.render("Você Perdeu com "+str(pontos) +
                         " pontos!", True, white)
    gameDisplay.blit(texto, (50, 100))
    fonteContinue = pygame.font.Font("freesansbold.ttf", 25)
    textoContinue = fonteContinue.render("press enter to restart", True, white)
    gameDisplay.blit(textoContinue, (50, 200))

    pygameDisplay.update()

def jogo():
    jogando = True
    movimentoX = 0
    movimentoY = random.randrange(0, altura)
    velocidade = 1
    direcao = True
    posicaoXPeixe = 450
    posicaoYPeixe = 100
    movimentoXPeixe = 0
    movimentoYPeixe = 0
    pontos = 0
    arpão = pygame.image.load("assets/arpão.png.jpeg")
    arpão = pygame.transform.flip(arpão, True, False)

    peixe = pygame.image.load("assets/peixe.png.jpeg")
    pygame.mixer.music.load("assets/trilha.mp3.mpeg")
    pygame.mixer.music.play(-1)
    pygame.mixer.music.set_volume(0.1)
    larguraPeixe = 100
    alturaPeixe = 100
    larguraArpão = 150
    alturaArpão = 52
    limiar = 28
    velocidadePeixe = 8

    while True:

        for event in gameEvents.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    movimentoXPeixe = -velocidadePeixe
                elif event.key == pygame.K_RIGHT:
                    movimentoXPeixe = velocidadePeixe
                elif event.key == pygame.K_UP:
                    movimentoYPeixe = -velocidadePeixe
                elif event.key == pygame.K_DOWN:
                    movimentoYPeixe = velocidadePeixe
                elif event.key == pygame.K_RETURN:
                    jogo()

            if event.type == pygame.KEYUP:
                if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT or event.key == pygame.K_UP or event.key == pygame.K_DOWN:
                    movimentoXPeixe = 0
                    movimentoYPeixe = 0

        if jogando == True:

            posicaoXPeixe = posicaoXPeixe + movimentoXPeixe
            if posicaoXPeixe < 0:
                posicaoXPeixe = 0
            elif posicaoXPeixe > largura - larguraPeixe:
                posicaoXPeixe = largura - larguraPeixe

            posicaoYPeixe = posicaoYPeixe + movimentoYPeixe
            if posicaoYPeixe < 0:
                posicaoYPeixe = 0
            elif posicaoYPeixe > altura - alturaPeixe:
                posicaoYPeixe = altura - alturaPeixe

            gameDisplay.fill(pink)
            gameDisplay.blit(bg, (0, 0))
            gameDisplay.blit(peixe, ( posicaoXPeixe, posicaoYPeixe ) )
            gameDisplay.blit(arpão, (movimentoX, movimentoY))

            if direcao == True:
                if movimentoX <= 800 - 150:
                    movimentoX = movimentoX + velocidade
                else:

                    direcao = False
                    pontos = pontos + 1
                    movimentoY = random.randrange(0, altura)
                    velocidade = velocidade + 1
                    arpão = pygame.transform.flip(arpão, True, False)
            else:
                if movimentoX >= 0:
                    movimentoX = movimentoX - velocidade
                else:

                    direcao = True
                    pontos += 1
                    movimentoY = random.randrange(0, altura)
                    velocidade = velocidade + 1
                    arpão = pygame.transform.flip(arpão, True, False)

            fonte = pygame.font.Font('freesansbold.ttf', 20)
            texto = fonte.render("Pontos: "+str(pontos), True, white)
            gameDisplay.blit(texto, (20, 280))

            pixelYArpão = list(range(movimentoY, movimentoY+alturaArpão + 1))
            pixelXArpão = list(range(movimentoX, movimentoX+larguraArpão + 1))

            pixelYPeixe = list(range(posicaoYPeixe, posicaoYPeixe + alturaPeixe +1 ))
            pixelXPeixe= list(range(posicaoXPeixe, posicaoXPeixe + larguraPeixe +1 ))

            colisaoY = list( set(pixelYArpão) & set(pixelYPeixe)  )
            colisaoX = list( set(pixelXArpão) & set(pixelXPeixe)  )
            if len(colisaoY) > limiar:
                if len(colisaoX) > limiar:
                    jogando =  False
                    dead(pontos)

        pygameDisplay.update()
        clock.tick(60)

jogo()


import xlwings as xw
import pandas as pd

#wb=xw.Book()
#wb.save('test.xlsx')
#wbtest=xw.Book('data.xlsx')
#wb1=wbtest.sheets['Sheet1']
#wb1.range('A1').value='hello'
wb =xw.Book('Class.xlsx')
sheet= wb.sheets("ClassA")
#rg=sheet.range('A').value
#sheet.range('C2').value=' helloxlwings'
df =sheet.range('A1:C3').options(pd.DataFrame).value    #import pandas as pd
#print(df)
df=df[:2]
xw.view(df)
wb.close


# -*- coding: utf-8 -*-

import json
from operator import itemgetter
from itertools import groupby
from datetime import datetime, date, timedelta

from jinja2 import Environment

from django.contrib.staticfiles.storage import staticfiles_storage
from django.shortcuts import get_object_or_404, get_list_or_404
from django.core.urlresolvers import reverse
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render, redirect
from django.db import models
from django.db.models import Sum, Count
from rest_framework import generics, filters
from rest_framework.decorators import api_view
from rest_framework.response import Response
import django_filters

from .serializers import ClockItemSerializer
from .models import ClockItem
from tms.utils import Utils
from myssg.items import Item

CATEGORIES = ['工作', '学习', '生活', '其他']
CATEGORIES_NAME_DICT = {u'工作': 'work', u'学习': 'study', u'生活': 'life', u'其他': 'other'}
WEEK_DAY_STR = ['周一', '周二', '周三', '周四', '周五', '周六', '周日']
MONTHS_STR = ['一月', '二月', '三月', '四月', '五月', '六月', '七月', '八月', '九月', '十月', '十一月', '十二月']

class ClockItemFilter(filters.FilterSet):
    min_st = django_filters.DateTimeFilter(name='start_time', lookup_expr='gte')
    max_st = django_filters.DateTimeFilter(name='start_time', lookup_expr='lt')

    class Meta:
        model = ClockItem
        fields = ['min_st', 'max_st', 'date', 'category', 'project', 'thing']

class ClockItemList(generics.ListAPIView):
    queryset = ClockItem.objects.all().order_by('start_time')
    serializer_class = ClockItemSerializer
    filter_class = ClockItemFilter

def index(request):
    cur_dt = datetime.now()
    tms_begin_date = TMS_BEGIN_DATE
    tms_end_date = cur_dt.date()

    # 各年简报
    years_stats_dict = dict()
    tms_data_group_by_year_category = ClockItem.objects.values('year', 'category').\
        annotate(Count('time_cost_min'), tc_sum=Sum('time_cost_min'))
    for item in tms_data_group_by_year_category:
        year = item['year']
        category = item['category']
        tc_sum = item['tc_sum']
        if year not in years_stats_dict:
            years_stats_dict[year] = {
                'year': year,
                'work_time': 0,
                'study_time': 0,
                'all_time': 0,
                'valid_time': 0,
            }
        year_stats = years_stats_dict[year]
        if category == '工作':
            year_stats['work_time'] += tc_sum
            year_stats['valid_time'] += tc_sum
        elif category == '学习':
            year_stats['study_time'] += tc_sum
            year_stats['valid_time'] += tc_sum
        year_stats['all_time'] += tc_sum
    years_stats = list(year_stats for (year, year_stats) in years_stats_dict.items())
    years_stats.sort(key=itemgetter('year'), reverse=True)
    for year_stats in years_stats:
        year = year_stats['year']

        # 计算各年总天数
        begin_date = datetime(year=year, month=1, day=1).date()
        end_date = datetime(year=year, month=12, day=31).date()
        if begin_date < tms_begin_date:
            begin_date = tms_begin_date
        if end_date > tms_end_date:
            end_date = tms_end_date
        days_num = (end_date - begin_date).days + 1
        year_stats['days_num'] = days_num
        year_stats['avg_valid_time'] = Utils.min_to_hour(float(year_stats['valid_time']) / float(days_num), 2)
        year_stats['all_time'] = Utils.min_to_hour(year_stats['all_time'])
        year_stats['work_time'] = Utils.min_to_hour(year_stats['work_time'])
        year_stats['study_time'] = Utils.min_to_hour(year_stats['study_time'])
        year_stats['valid_time'] = Utils.min_to_hour(year_stats['valid_time'])

    # 本年各月简报
    cur_year = cur_dt.year
    months_stats_dict = dict()
    for month in range(1, 13):
        months_stats_dict[month] = {
            'month': month,
            'month_str': MONTHS_STR[month - 1],
            'work_time': 0,
            'study_time': 0,
            'all_time': 0,
            'valid_time': 0,
        }
    cur_year_data_group_by_month_category = ClockItem.objects.filter(year=cur_year).values('month', 'category'). \
        annotate(Count('time_cost_min'), tc_sum=Sum('time_cost_min'))
    for item in cur_year_data_group_by_month_category:
        month = item['month']
        category = item['category']
        tc_sum = item['tc_sum']

        month_stats = months_stats_dict[month]
        if category == '工作':
            month_stats['work_time'] += tc_sum
            month_stats['valid_time'] += tc_sum
        elif category == '学习':
            month_stats['study_time'] += tc_sum
            month_stats['valid_time'] += tc_sum
        month_stats['all_time'] += tc_sum
    months_stats = list(month_stats for (month, month_stats) in months_stats_dict.items())
    months_stats.sort(key=itemgetter('month'))
    for month_stats in months_stats:
        month = month_stats['month']

        # 计算本年各月总天数
        begin_date = datetime(year=cur_year, month=month, day=1).date()
        if month == 12:
            end_date = datetime(year=cur_year, month=12, day=31).date()
        else:
            end_date = datetime(year=cur_year, month=month + 1, day=1).date() - timedelta(days=1)
        if begin_date < tms_begin_date:
            begin_date = tms_begin_date
        if end_date > tms_end_date:
            end_date = tms_end_date
        days_num = (end_date - begin_date).days + 1
        if days_num < 0:
            days_num = 0
            month_stats['avg_valid_time'] = 0
            month_stats['valid_time'] = 0
        else:
            month_stats['avg_valid_time'] = Utils.min_to_hour(float(month_stats['valid_time']) / float(days_num), 2)
            month_stats['valid_time'] = Utils.min_to_hour(month_stats['valid_time'])
        month_stats['days_num'] = days_num
        month_stats['all_time'] = Utils.min_to_hour(month_stats['all_time'])
        month_stats['work_time'] = Utils.min_to_hour(month_stats['work_time'])
        month_stats['study_time'] = Utils.min_to_hour(month_stats['study_time'])

    # 本日的统计
    today_work_time = 0
    today_study_time = 0
    today_data_group_by_month_category = ClockItem.objects.filter(date=cur_dt).values('category'). \
        annotate(Count('time_cost_min'), tc_sum=Sum('time_cost_min'))
    for item in today_data_group_by_month_category:
        category = item['category']
        tc_sum = item['tc_sum']
        if category == '工作':
            today_work_time += tc_sum
        elif category == '学习':
            today_study_time += tc_sum
    today_work_time = Utils.min_to_hour(today_work_time)
    today_study_time = Utils.min_to_hour(today_study_time)

    clock_items = ClockItem.objects.filter(date=cur_dt).order_by('-start_time')
    latest_clock_item = None
    if len(clock_items) >= 1:
        latest_clock_item = clock_items[0]

    return render(request, 'tms/index.html', {
        'years_stats': years_stats,
        'months_stats': months_stats,
        'cur_dt': cur_dt,
        'cur_year': cur_dt.year,
        'cur_month': cur_dt.month,
        'today_work_time': today_work_time,
        'today_study_time': today_study_time,
        'latest_clock_item': latest_clock_item

    })

@api_view(['GET'])
def api_day_stats(request):
    date_str = request.GET['date']
    if 'days_num' in request.GET:
        days_num = int(request.GET['days_num'])
    else:
        days_num = 1
    min_date = datetime.strptime(date_str, '%Y-%m-%d').date()
    max_date = min_date + timedelta(days=days_num)
    clock_items = ClockItem.objects.filter(start_time__gte=min_date, end_time__lt=max_date).order_by('start_time')
    days_stats = generate_days_stats(clock_items, min_date, days_num)
    report = generate_report(clock_items, days_num)
    return Response({
        'days_stats': days_stats,
        'report': report,
        'clock_items': ClockItemSerializer(clock_items, many=True).data,
    })

DEFAULT_MIN_DT = datetime(year=1970, month=1, day=1)
DEFAULT_MAX_DT = datetime(year=9999, month=12, day=31)
TMS_BEGIN_DATE = date(year=2015, month=2, day=20)

@api_view(['GET'])
def api_project_stats(request):
    if 'min_dt' in request.GET:
        min_dt = Utils.parse_dt_str(request.GET['min_dt'])
    else:
        min_dt = DEFAULT_MIN_DT
    if 'max_dt' in request.GET:
        max_dt = Utils.parse_dt_str(request.GET['max_dt'])
    else:
        max_dt = DEFAULT_MAX_DT

    category = request.GET['c']
    project = request.GET['p']
    thing = request.GET.get('t', None)
    clock_items = ClockItem.objects.filter(start_time__gte=min_dt, end_time__lt=max_dt).order_by('-start_time')
    if category is not None:
        clock_items = clock_items.filter(category=category)
    if project is not None:
        clock_items = clock_items.filter(project=project)
    if thing is not None:
        clock_items = clock_items.filter(thing=thing)
    clock_items = clock_items.order_by('start_time')
    if min_dt == DEFAULT_MIN_DT:
        min_dt = clock_items[0].start_time
    if max_dt == DEFAULT_MAX_DT:
        last_clock_item = clock_items[len(clock_items) - 1]
        print(last_clock_item.end_time, last_clock_item.thing)
        max_dt = last_clock_item.end_time

    days_num = (max_dt.date() - min_dt.date()).days + 1
    print(min_dt, max_dt, days_num)
    # if days_num > 500:
    if days_num > 200:
        # Months stats
        interval = 'month'
        stats = generate_months_stats(clock_items, min_dt.date(), max_dt.date())
    elif days_num > 100:
        # Weeks stats
        interval = 'week'
        stats = generate_weeks_stats(clock_items, min_dt.date(), max_dt.date())
    else:
        # Days stats
        interval = 'day'
        stats = generate_days_stats(clock_items, min_dt.date(), days_num, with_hours_stats=False)

    return Response({
        'min_dt': min_dt,
        'max_dt': max_dt,
        'category': category,
        'interval': interval,
        'stats': stats,
    })

def generate_months_stats(clock_items, min_date, max_date):
    first_first_date = date(year=min_date.year, month=min_date.month, day=1)
    days_stats_dict = dict()
    first_date = first_first_date
    while True:
        interval_id = Utils.get_month_id(first_date)
        days_stats_dict[interval_id] = {
            'id': interval_id,
            'date': first_date,
            'all_time': 0,
            'valid_time': 0,
            'work_time': 0,
            'study_time': 0,
            'items_num': 0,
            }
        one_day_next_month = first_date + timedelta(days=32)
        first_date = date(year=one_day_next_month.year, month=one_day_next_month.month, day=1)
        if first_date > max_date:
            break

    # Fill days_stats_dict
    for clock_item in clock_items:
        dt = clock_item.date
        interval_id = Utils.get_month_id(dt)
        time_cost_min = clock_item.time_cost_min
        days_stats_dict[interval_id]['all_time'] += time_cost_min
        days_stats_dict[interval_id]['items_num'] += 1
        category = clock_item.category
        if category == '工作':
            days_stats_dict[interval_id]['work_time'] += time_cost_min
            days_stats_dict[interval_id]['valid_time'] += time_cost_min
        elif category == '学习':
            days_stats_dict[interval_id]['study_time'] += time_cost_min
            days_stats_dict[interval_id]['valid_time'] += time_cost_min

    weeks_stats = list(days_stats_dict.values())
    weeks_stats.sort(key=itemgetter('date'))

    return weeks_stats

def generate_weeks_stats(clock_items, min_date, max_date):
    first_monday_date = min_date - timedelta(days=min_date.weekday())
    days_stats_dict = dict()
    monday_date = first_monday_date
    while True:
        week_id = Utils.get_week_id(monday_date)
        days_stats_dict[week_id] = {
            'id': week_id,
            'date': monday_date,
            'all_time': 0,
            'valid_time': 0,
            'work_time': 0,
            'study_time': 0,
            'items_num': 0,
            }
        monday_date += timedelta(days=7)
        if monday_date > max_date:
            break

    # Fill days_stats_dict
    for clock_item in clock_items:
        dt = clock_item.date
        week_id = Utils.get_week_id(dt)
        # print(dt)
        time_cost_min = clock_item.time_cost_min
        days_stats_dict[week_id]['all_time'] += time_cost_min
        days_stats_dict[week_id]['items_num'] += 1
        category = clock_item.category
        if category == '工作':
            days_stats_dict[week_id]['work_time'] += time_cost_min
            days_stats_dict[week_id]['valid_time'] += time_cost_min
        elif category == '学习':
            days_stats_dict[week_id]['study_time'] += time_cost_min
            days_stats_dict[week_id]['valid_time'] += time_cost_min

    weeks_stats = list(days_stats_dict.values())
    weeks_stats.sort(key=itemgetter('date'))

    return weeks_stats

def generate_days_stats(clock_items, min_date, days_num,
                        with_hours_stats=True, with_sparkline_data=False):
    days_stats_dict = dict()
    for i in range(days_num):
        dt = min_date + timedelta(days=i)
        days_stats_dict[dt] = {
            'date': dt,
            'date_str': dt.strftime('%Y-%m-%d'),
            'week_day': WEEK_DAY_STR[dt.weekday()],
            'all_time': 0,
            'valid_time': 0,
            'work_time': 0,
            'study_time': 0,
            'items_num': 0,
        }
        if with_hours_stats:
            days_stats_dict[dt]['hours_stats'] = [{'work': 0, 'study': 0, 'life': 0, 'other': 0} for i in range(24)]

    # Fill days_stats_dict
    for clock_item in clock_items:
        dt = clock_item.date
        # print(dt)
        time_cost_min = clock_item.time_cost_min
        days_stats_dict[dt]['all_time'] += time_cost_min
        days_stats_dict[dt]['items_num'] += 1
        category = clock_item.category
        if category == '工作':
            days_stats_dict[dt]['work_time'] += time_cost_min
            days_stats_dict[dt]['valid_time'] += time_cost_min
        elif category == '学习':
            days_stats_dict[dt]['study_time'] += time_cost_min
            days_stats_dict[dt]['valid_time'] += time_cost_min

        if with_hours_stats:
            # Hour stats
            hours_stats = days_stats_dict[dt]['hours_stats']
            start_time = clock_item.start_time
            end_time = clock_item.end_time
            start_hour = start_time.hour
            end_hour = end_time.hour
            category_name = CATEGORIES_NAME_DICT[clock_item.category]
            if start_hour == end_hour:
                hours_stats[start_hour][category_name] += end_time.minute - start_time.minute
            else:
                if end_hour < start_hour:
                    end_hour += 24
                for hour in range(start_hour, end_hour + 1):
                    if hour >= 24:
                        next_dt = dt + timedelta(days=1)
                        if next_dt not in days_stats_dict:
                            continue
                        hours_stats = days_stats_dict[next_dt]['hours_stats']
                        hour -= 24
                    if hour == start_hour:
                        hours_stats[hour][category_name] += 60 - start_time.minute
                    elif hour == end_hour or hour == end_hour - 24:
                        hours_stats[hour][category_name] += end_time.minute
                    else:
                        hours_stats[hour][category_name] += 60

    days_stats = list(days_stats_dict.values())
    days_stats.sort(key=itemgetter('date'), reverse=True)

    # 额外信息
    for day_stats in days_stats:
        day_stats['work_time_str'] = Utils.min_to_hour2(day_stats['work_time'])
        day_stats['study_time_str'] = Utils.min_to_hour2(day_stats['study_time'])
        day_stats['valid_time_str'] = Utils.min_to_hour2(day_stats['valid_time'])
        day_stats['all_time_str'] = Utils.min_to_hour2(day_stats['all_time'])

    # Sparkline data
    if with_sparkline_data:
        # Generate sparkline data, see http://omnipotent.net/jquery.sparkline/
        for day_stats in days_stats:
            sparkline_data = '['
            for hour_stats in day_stats['hours_stats']:
                sparkline_data += '[%d, %d, %d, %d], ' % (hour_stats['work'], hour_stats['study'], hour_stats['life'], hour_stats['other'])
            sparkline_data += ']'
            day_stats['sparkline_data'] = sparkline_data

    return days_stats

def generate_report(clock_items, days_num):
    """
    Sample data
        report_data = {
            'categories': [
                {
                    'name': '工作',
                    'projects': [
                        {
                            'name': 'XX项目',
                            'things': [
                                {
                                    'name': 'XXX模块开发',
                                    'time_cost': 30,
                                    },

                                ]
                        }
                    ]
                }
            ]
        }
    """
    categories_data = list()
    project_id = 1
    total_cost = 0
    for clock_item in clock_items:
        category = clock_item.category
        project = clock_item.project
        thing = clock_item.thing
        time_cost_min = clock_item.time_cost_min
        total_cost += time_cost_min

        # Find category
        category_data = None
        for it in categories_data:
            if it['name'] == category:
                category_data = it
                break
        if category_data is None:
            category_data = {
                'name': category,
                'cost': time_cost_min
            }
            categories_data.append(category_data)
        else:
            category_data['cost'] += time_cost_min

        if 'projects' not in category_data:
            category_data['projects'] = list()
        projects_data = category_data['projects']

        # Find project
        project_data = None
        for it in projects_data:
            if it['name'] == project:
                project_data = it
                break
        if project_data is None:
            project_data = {
                'name': project,
                'id': project_id,
                'cost': time_cost_min
            }
            project_id += 1
            projects_data.append(project_data)
        else:
            project_data['cost'] += time_cost_min

        if 'things' not in project_data:
            project_data['things'] = list()
        things_data = project_data['things']

        # Find thing
        thing_data = None
        for it in things_data:
            if it['name'] == thing:
                thing_data = it
                break
        if thing_data is None:
            thing_data = {
                'name': thing,
                'cost': time_cost_min
            }
            things_data.append(thing_data)
        else:
            thing_data['cost'] += time_cost_min

    # Calculate percentage
    minutes_of_days = days_num * 24 * 60
    for category_data in categories_data:
        category_data['pct'] = "{0:.1f}%".format(float(category_data['cost'])/minutes_of_days * 100)
        category_time_cost = category_data['cost']
        for project_data in category_data['projects']:
            print(u'' + category_data['name'], category_time_cost)
            project_data['pct'] = "{0:.1f}%".format(float(project_data['cost'])/category_time_cost * 100)

    # Sort
    categories_data.sort(key=lambda x: CATEGORIES.index(x['name']))
    for category_data in categories_data:
        projects_data = category_data['projects']
        projects_data.sort(key=itemgetter('cost'), reverse=True)
        for project_data in projects_data:
            thing_data = project_data['things']
            thing_data.sort(key=itemgetter('cost'), reverse=True)

    # Change form of time cost
    for category_data in categories_data:
        category_data['cost'] = Utils.min_to_hour2(category_data['cost'])
        for project_data in category_data['projects']:
            project_data['cost'] = Utils.min_to_hour2(project_data['cost'])

    report_data = {
        'categories': categories_data,
        'total_cost': total_cost
    }

    return report_data

@api_view(['GET'])
def api_week_stats(request):
    iso_year = int(request.GET['year'])
    week = int(request.GET['week'])
    week_data_group_by_category = ClockItem.objects.filter(iso_year=iso_year, week=week). \
        values('category').annotate(Count('id'), tc_sum=Sum('time_cost_min')). \
        order_by('-tc_sum')
    week_data_group_by_category = list((i['category'], i['id__count'], i['tc_sum'])
                                       for i in week_data_group_by_category)
    week_data_group_by_project = ClockItem.objects.filter(iso_year=iso_year, week=week). \
        values('project', 'category').annotate(Count('id'), tc_sum=Sum('time_cost_min')). \
        order_by('-tc_sum')

    week_data_group_by_project = list((i['category'], i['project'], i['id__count'], i['tc_sum'])
                                      for i in week_data_group_by_project)

    legend_data = list(t[0] for t in week_data_group_by_category)
    legend_data.sort(key=lambda x: CATEGORIES.index(x))
    week_data_group_by_category.sort(key=lambda x: CATEGORIES.index(x[0]))
    week_data_group_by_project.sort(key=lambda x: legend_data.index(x[0]))
    inner_data = list({'name': t[0], 'value': t[2]} for t in week_data_group_by_category)
    outer_data = list({'name': t[1], 'value': t[3]} for t in week_data_group_by_project)
    return Response({
        'legend': legend_data,
        'inner': inner_data,
        'outer': outer_data,
        })

def get_project(request):
    category = request.GET['c']
    project = request.GET['p']
    clock_items = ClockItem.objects.filter(category=category, project=project).order_by('start_time')
    cis_num = len(clock_items)
    start_time = clock_items[0].start_time
    end_time = clock_items[cis_num - 1].end_time
    days_num = (end_time.date() - start_time.date()).days + 1
    return render(request, 'tms/project.html', {
        'category': category,
        'project': project,
        'clock_items': clock_items,
        'start_time': start_time,
        'end_time': end_time,
        'days_num': days_num
    })

def all_projects(request):
    year = None
    all_projects_stats = ClockItem.objects
    if year is not None:
        all_projects_stats = all_projects_stats.filter(year=year)
    all_projects_stats = all_projects_stats.values('category', 'project'). \
        annotate(tc_count=models.Count('id'), tc_sum=models.Sum('time_cost_min'),
                 start_time_min=models.Min('start_time'), end_time_max=models.Max('end_time')). \
        order_by('-tc_sum')
    for project_stats in all_projects_stats:
            project_stats['days_num'] = \
            (project_stats['end_time_max'].date() - project_stats['start_time_min'].date()).days + 1
    return render(request, 'tms/all_projects.html', {
        'all_projects_stats': all_projects_stats,
        })

def time_analyzer(request):
    category = request.GET['c']
    project = request.GET['p']
    clock_items = ClockItem.objects.filter(category=category, project=project).order_by('start_time')
    cis_num = len(clock_items)
    start_time = clock_items[0].start_time
    end_time = clock_items[cis_num - 1].end_time
    return render(request, 'tms/time_analyzer.html', {
        'category': category,
        'project': project,
        'clock_items': clock_items,
        'start_time': start_time,
        'end_time': end_time,
    })

def calendar(request):
    return render(request, 'tms/calendar.html')

def get_report(request):
    dt = datetime.now()
    iso_year, week, weekday = dt.isocalendar()
    year = dt.year
    month = dt.month

    clock_items = ClockItem.objects.filter(iso_year=iso_year).all()
    return render(request, 'tms/report.html', {
        'clock_items': clock_items,
    })

def get_monday_date_of_iso_week(iso_year, week):
    ret = datetime.strptime('%04d-%02d-1' % (iso_year, week), '%Y-%W-%w')
    if date(iso_year, 1, 4).isoweekday() > 4:
        ret -= timedelta(days=7)
    return ret

def day_report(request):
    if 'date' in request.GET:
        return day_report_detail(request, request.GET['date'])

    if 'num' in request.GET:
        days_num = int(request.GET['num'])
    else:
        days_num = 31
    cur_date = datetime.now().date()
    min_date = cur_date - timedelta(days=days_num - 1)
    clock_items = ClockItem.objects.filter(date__gte=min_date)
    days_stats = generate_days_stats(clock_items, min_date, days_num, with_sparkline_data=True)

    return render(request, 'tms/day_report.html', {
        'days_stats': days_stats
        })

def day_report_detail(request, date_str):
    cur_date_str = datetime.now().strftime('%Y-%m-%d')
    if date_str == cur_date_str:
        is_cur_day = True
    else:
        is_cur_day = False
    dt = datetime.strptime(date_str, '%Y-%m-%d')
    prev_date_str = (dt - timedelta(days=1)).strftime('%Y-%m-%d')
    next_date_str = (dt + timedelta(days=1)).strftime('%Y-%m-%d')

    clock_items = ClockItem.objects.filter(date=date_str).order_by('start_time')
    report_data = generate_report(clock_items, 1)
    return render(request, 'tms/day_report_detail.html', {
        'is_cur_day': is_cur_day,
        'date_str': date_str,
        'date': dt,
        'clock_items': clock_items,
        'prev_date_str': prev_date_str,
        'next_date_str': next_date_str,
        'report_data': report_data,
        })

def week_report(request):
    cur_iso_year, cur_week, weekday = datetime.now().isocalendar()
    if 'year' not in request.GET and 'week' not in request.GET:
        iso_year = cur_iso_year
        week = cur_week
    else:
        iso_year = int(request.GET['year'])
        week = int(request.GET['week'])

    if iso_year == cur_iso_year and week == cur_week:
        is_cur_week = True
    else:
        is_cur_week = False

    dt_of_this_monday = get_monday_date_of_iso_week(iso_year, week)
    dt_of_prev_monday = dt_of_this_monday - timedelta(days=7)
    dt_of_next_monday = dt_of_this_monday + timedelta(days=7)
    iso_year_of_prev_week, prev_week, weekday = dt_of_prev_monday.isocalendar()
    iso_year_of_next_week, next_week, weekday = dt_of_next_monday.isocalendar()

    clock_items = ClockItem.objects.filter(iso_year=iso_year, week=week).order_by('start_time')
    clock_items_list = list()
    for dt, clock_items_of_day in groupby(clock_items, itemgetter('date')):
        clock_items_list.append({
            'date': dt,
            'clock_items': list(clock_items_of_day),
        })

    print('This monday: ', dt_of_this_monday)
    report_data = generate_report(clock_items, 7)
    days_stats = generate_days_stats(clock_items, min_date=dt_of_this_monday.date(),
                                     days_num=7, with_sparkline_data=True)
    days_stats.sort(key=itemgetter('date'))

    return render(request, 'tms/week_report.html', {
        'is_cur_week': is_cur_week,
        'iso_year': iso_year,
        'week': week,
        'clock_items': clock_items,
        'clock_items_list': clock_items_list,
        'prev_week': prev_week,
        'iso_year_of_prev_week': iso_year_of_prev_week,
        'next_week': next_week,
        'iso_year_of_next_week': iso_year_of_next_week,
        'report_data': report_data,
        'days_stats': days_stats,
        # 'daily_overview': daily_overview,
    })

def custom_report(request):
    # 默认取上周五下午六点到本周五下午六点作为start_time和end_time
    today = datetime.now().date()
    # 获取本周五18点的datetime
    end_dt_of_work_week = datetime(year=today.year, month=today.month, day=today.day,
                                   hour=18, minute=0, second=0)
    end_dt_of_work_week += timedelta(4 - today.weekday())
    # 获取上周四14点的datetime
    begin_dt_of_work_week = end_dt_of_work_week - timedelta(weeks=1)

    start_time_str = request.GET.get('start', None)
    if start_time_str is None:
        start_time = begin_dt_of_work_week
    else:
        start_time = datetime.strptime(start_time_str, '%Y%m%d')
    start_time_str = start_time.strftime('%Y-%m-%d %H:%M')

    end_time_str = request.GET.get('end', None)
    if end_time_str is None:
        end_time = end_dt_of_work_week
    else:
        end_time = datetime.strptime(end_time_str, '%Y%m%d') + timedelta(days=1)
    end_time_str = end_time.strftime('%Y-%m-%d %H:%M')

    clock_items = ClockItem.objects.filter(start_time__gte=start_time, end_time__lt=end_time).order_by('start_time')
    days_num = (end_time - start_time).days
    report_data = generate_report(clock_items, days_num)

    return render(request, 'tms/custom_report.html', {
        'start_time_str': start_time_str,
        'end_time_str': end_time_str,
        'days_num': days_num,
        'clock_items': clock_items,
        'report_data': report_data,
        })

def year_report(request):
    tms_begin_date = TMS_BEGIN_DATE
    tms_end_date = datetime.now().date()
    cur_year = datetime.now().year
    if 'year' not in request.GET:
        year = cur_year
    else:
        year = int(request.GET['year'])
    clock_items = ClockItem.objects.filter(year=year).order_by('start_time')

    # 总体统计
    total_stats = dict()
    # 计算总天数
    begin_date = datetime(year=year, month=1, day=1).date()
    end_date = datetime(year=year, month=12, day=31).date()
    if begin_date < tms_begin_date:
        begin_date = tms_begin_date
    if end_date > tms_end_date:
        end_date = tms_end_date
    days_num = (end_date - begin_date).days + 1
    total_stats['days_num'] = days_num

    # 计算各项总时间
    all_time = 0
    work_time = 0
    study_time = 0
    valid_time = 0
    for clock_item in clock_items:
        time_cost_min = clock_item.time_cost_min
        category = clock_item.category
        all_time += time_cost_min
        if category in ['工作', '学习']:
            valid_time += time_cost_min
        if category in ['工作']:
            work_time += time_cost_min
        if category in ['学习']:
            study_time += time_cost_min
    total_stats['all_time'] = Utils.min_to_hour(all_time)
    total_stats['work_time'] = Utils.min_to_hour(work_time)
    total_stats['study_time'] = Utils.min_to_hour(study_time)
    total_stats['valid_time'] = Utils.min_to_hour(valid_time)
    total_stats['avg_valid_time'] = Utils.min_to_hour(float(valid_time) / float(days_num), 2)

    # 生成报表
    report_data = generate_report(clock_items, days_num)

    return render(request, 'tms/year_report.html', {
        'year': year,
        'begin_date': begin_date,
        'end_date': end_date,
        'clock_items': clock_items,
        'report_data': report_data,
        'total_stats': total_stats,
    })

def week_stats(request):
    iso_year = int(request.GET['year'])
    week = int(request.GET['week'])
    week_data_group_by_category = ClockItem.objects.filter(iso_year=iso_year, week=week).\
        values('category').annotate(Count('id'), tc_sum=Sum('time_cost_min')).\
        order_by('-tc_sum')
    week_data_group_by_category = list((i['category'], i['id__count'], i['tc_sum'])
                                       for i in week_data_group_by_category)
    week_data_group_by_project = ClockItem.objects.filter(iso_year=iso_year, week=week). \
        values('project', 'category').annotate(Count('id'), tc_sum=Sum('time_cost_min')).\
        order_by('-tc_sum')

    week_data_group_by_project = list((i['category'], i['project'], i['id__count'], i['tc_sum'])
                                      for i in week_data_group_by_project)

    legend_data = list(t[0] for t in week_data_group_by_category)
    legend_data.sort(key=lambda x: CATEGORIES.index(x))
    week_data_group_by_category.sort(key=lambda x: CATEGORIES.index(x[0]))
    week_data_group_by_project.sort(key=lambda x: legend_data.index(x[0]))
    inner_data = list({'name': t[0], 'value': t[2]} for t in week_data_group_by_category)
    outer_data = list({'name': t[1], 'value': t[3]} for t in week_data_group_by_project)
    return JsonResponse({
        'legend': legend_data,
        'inner': inner_data,
        'outer': outer_data,
    })

def year_stats_step_by_month_and_week(request):
    """
    data = {
        'labels': ['一月', '二月', '三月', '...'],
        'valid_time': [15, 50, 33, ...],
        'work_time': [1.7, 25, 22, ...],
        'study_time': [13, 25, 11, ...],
    }
    """
    year = int(request.GET['year'])
    time_cost_group_by_month_category = ClockItem.objects.filter(year=year). \
        values('month', 'category').annotate(Sum('time_cost_min'), Count('time_cost_min'))

    valid_time_of_month = [0] * 12
    work_time_of_month = [0] * 12
    study_time_of_month = [0] * 12
    for item in time_cost_group_by_month_category:
        category = item['category']
        month = item['month']
        time_cost_min = item['time_cost_min__sum']
        if category == '工作':
            work_time_of_month[month - 1] += time_cost_min
            valid_time_of_month[month - 1] += time_cost_min
        elif category == '学习':
            study_time_of_month[month - 1] += time_cost_min
            valid_time_of_month[month - 1] += time_cost_min
    month_labels = ['一月', '二月', '三月', '四月', '五月', '六月', '七月', '八月', '九月', '十月', '十一月', '十二月']
    valid_time_of_month = ['%.1f' % (float(i) / 60.0) for i in valid_time_of_month]
    work_time_of_month = ['%.1f' % (float(i) / 60.0) for i in work_time_of_month]
    study_time_of_month = ['%.1f' % (float(i) / 60.0) for i in study_time_of_month]

    # Get last week of year
    time_cost_group_by_week_category = ClockItem.objects.filter(iso_year=year). \
        values('week', 'category').annotate(Sum('time_cost_min'), Count('time_cost_min'))
    last_day_of_year = datetime(year=year, month=12, day=31)
    while True:
        last_week = last_day_of_year.isocalendar()[1]
        if last_week != 1:
            break
        else:
            last_day_of_year -= timedelta(days=1)

    valid_time_of_week = [0] * last_week
    work_time_of_week = [0] * last_week
    study_time_of_week = [0] * last_week
    for item in time_cost_group_by_week_category:
        category = item['category']
        week = item['week']
        time_cost_min = item['time_cost_min__sum']
        if category == '工作':
            work_time_of_week[week - 1] += time_cost_min
            valid_time_of_week[week - 1] += time_cost_min
        elif category == '学习':
            study_time_of_week[week - 1] += time_cost_min
            valid_time_of_week[week - 1] += time_cost_min
    week_labels = ['W%d' % w for w in range(1, last_week + 1)]
    valid_time_of_week = ['%.1f' % (float(i) / 60.0) for i in valid_time_of_week]
    work_time_of_week = ['%.1f' % (float(i) / 60.0) for i in work_time_of_week]
    study_time_of_week = ['%.1f' % (float(i) / 60.0) for i in study_time_of_week]

    data = {
        'month': {
            'labels': month_labels,
            'valid_time': valid_time_of_month,
            'work_time': work_time_of_month,
            'study_time': study_time_of_month,
            },
        'week': {
            'labels': week_labels,
            'valid_time': valid_time_of_week,
            'work_time': work_time_of_week,
            'study_time': study_time_of_week,
        }
    }

    return JsonResponse(data)


#coding:utf-8
import torch
import torch.utils.data as data
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import os,re,jieba
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from config import Config
from multi_proc_utils import parallelize
from sklearn.preprocessing import MultiLabelBinarizer as MLB
from tqdm import tqdm
import pickle

config = Config()

""" 一：对样本和标签进行数值化，建立词表 """
def build_dataset(config):

    """ 1：加载数据 """
    print("\nLoading the dataset ... \n")
    point_df = pd.read_csv(config.point_path,header=None)
    point_df.dropna(inplace=True)
    point_df = point_df.rename(columns={0:"label",1:"content"})
    print(f"\nThe shape of the dataset : {point_df.shape}\n")

    """ 2：开多进程进行数据清洗和分词 """
    print("\nCleaning text and segmenting ... \n")
    point_df = parallelize(point_df,proc)

    """ 3：对样本进行 zero pad，并转化为id """
    print("\nZero padding and transfering id ...\n")
    text_tokenizer = Tokenizer(oov_token="<unk>")
    text_tokenizer.fit_on_texts(point_df["content"])
    corpus_x = text_tokenizer.texts_to_sequences(point_df["content"])
    config.max_len = calcu_max_len(point_df)
    corpus_x = pad_sequences(corpus_x,maxlen=config.max_len,padding="post",truncating="post")

    """ 4: 对多标签分类的标签进行数值化 """
    print("\nNumeralizing the multiclass labels ... \n")
    point_df["label"] = point_df["label"].apply(lambda x:x.split())
    mlb = MLB()
    corpus_y = mlb.fit_transform(point_df["label"])
    config.num_classes = corpus_y.shape[1]

    """ 5: 用样本构建词表，加入 <pad> 的id """
    print("\nBuilding the vocab ...\n")
    word_index = text_tokenizer.word_index
    vocab = dict({"<pad>":0}, **word_index)
    config.vocab_size = len(vocab)

    """ 保存好标签转化器，在模型预测时用 """
    save_pickle(mlb, config.mlb_path)

    """ 保存好词表，模型预测时用 """
    save_pickle(vocab,config.vocab_path)

    return corpus_x,corpus_y,vocab

""" 加载停用词 """
def load_stop_words(stop_word_path):
    file = open(stop_word_path, 'r', encoding='utf-8')
    stop_words = file.readlines()
    stop_words = [stop_word.strip() for stop_word in stop_words]
    return stop_words

stop_words = load_stop_words(config.stopwords_path)

""" 清洗文本 """
def clean_sentence(line):
    line = re.sub(
            "[a-zA-Z0-9]|[\s+\-\|\!\/\[\]\{\}_,.$%^*(+\"\')]+|[:：+——()?【】《》“”！，。？、~@#￥%……&*（）]+|题目", '',line)
    words = jieba.lcut(line, cut_all=False)
    return words

""" 进行分词 """
def sentence_proc(sentence):
    words = clean_sentence(sentence)
    words = [word for word in words if word not in stop_words]
    return ' '.join(words)

def proc(df):
    df["content"] = df["content"].apply(sentence_proc)
    return df

""" 计算输入长度，使其能涵盖95%样本 """
def calcu_max_len(df):

    df["lengths"] = df["content"].apply(lambda x:x.count(' ')+1)
    max_lengths = max(df["lengths"])
    for len_ in range(50,max_lengths,50):
        bool_ = df["lengths"] < len_
        cover_rate = sum(bool_.apply(int)) / len(bool_)
        if cover_rate >= 0.95:
            return len_

""" 四：计算类别权重，缓解类别不平衡问题 """
def calcu_class_weights(labels,config):
    labels = torch.FloatTensor(labels)

    freqs = torch.zeros_like(labels[0])
    for y in labels:
        freqs += y

    weights = freqs / len(labels)
    weights = 1 / torch.log(1.01 + weights)

    weights = weights.to(config.device)
    return weights

""" 二: 加载预训练词向量，并与词表相对应 """
def load_embed_matrix(vocab,config):

    """ 1: 加载百度百科词向量 """
    print("\nLoading baidu baike word2vec ...\n")
    embed_index = load_w2v(config.w2v_path)

    """ 2: 词向量矩阵与词表相对应 """
    vocab_size = len(vocab)
    embed_matrix = np.zeros((vocab_size,config.embed_dim))
    for word,index in vocab.items():
        vector = embed_index.get(word)
        if vector is not None:
            embed_matrix[index] = vector

    embed_matrix = torch.FloatTensor(embed_matrix)

    return embed_matrix

""" 加载百度百科词向量 """
def load_w2v(path):

    file = open(path,encoding="utf-8")

    embed_index = {}
    for i,line in tqdm(enumerate(file)):
        if i == 0:
            continue
        value = line.split()
        word = value[0]
        emb = np.asarray(value[1:], dtype="float32")
        embed_index[word] = emb

    return embed_index

""" 三: 划分数据集，并生成 batch 迭代器 """
def batch_generator(x,y,size,config):

    """ 1: 划分数据集 """
    print("\nSpliting the dataset ... \n")
    train_data, valid_data, test_data = split_dataset(x, y, size, config.device)

    """ 2: 生成 batch 迭代器 """
    train_iter = batch_iterator(train_data, config.batch_size)
    valid_iter = batch_iterator(valid_data, config.batch_size)
    test_iter = batch_iterator(test_data, config.batch_size)

    return train_iter,valid_iter,test_iter

""" 划分数据集 """
def split_dataset(x,y,size,device):

    train_x, valid_x, train_y, valid_y = train_test_split(x,y,test_size=size,random_state=10)
    train_x, test_x, train_y, test_y = train_test_split(train_x, train_y, test_size=size,random_state=10)

    train_x = torch.LongTensor(train_x).to(device)
    valid_x = torch.LongTensor(valid_x).to(device)
    test_x = torch.LongTensor(test_x).to(device)

    train_y = torch.FloatTensor(train_y).to(device)
    valid_y = torch.FloatTensor(valid_y).to(device)
    test_y = torch.FloatTensor(test_y).to(device)

    return (train_x,train_y) , (valid_x,valid_y), (test_x, test_y)

""" 重写 dataset 类"""
class MyDataset(data.Dataset):
    def __init__(self, dataset):
        self.X, self.Y = dataset

    def __getitem__(self, index):
        x, y = self.X[index], self.Y[index]
        return x, y

    def __len__(self):
        return len(self.X)

""" 生成 迭代器 """
def batch_iterator(dataset,batch_size):
    dataset = MyDataset(dataset)
    batcher = data.DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True)
    return batcher

""" 保存为pickle对象 """
def save_pickle(s,file_path):
    with open(file_path,'wb') as f:
        pickle.dump(s,f,protocol=2)

""" 加载pickle对象 """
def load_pickle(file_path):
    with open(file_path,'rb') as f:
        s = pickle.load(f)
    return s

if __name__ == "__main__":

    corpus_x, corpus_y, vocab = build_dataset(config)

    class_weights = calcu_class_weights(corpus_y,config)

    embed_matrix = load_embed_matrix(vocab,config)

    train_iter, valid_iter, test_iter = batch_generator(corpus_x, corpus_y, 0.15, config)
    for x_batch, y_batch in train_iter:
        print(f"The shape of content batch is {x_batch.shape}")
        print(f"The shape of label batch is {y_batch.shape}")
        break


import httplib
import json
import argparse

import vmware.common.constants as constants
import vmware.common.global_config as global_config

pylogger = global_config.pylogger
CAT_API_SERVER = 'cat-api.eng.vmware.com'

def retry_testrun(workunit_id=None, deliverable_id=None, api_key=None,
                  username=None):
    # Use this method only when absolutely necessary.
    # CAT team is not willing to allow more frequent reruns until the new UI
    # is ready. Also, this is not recommended due to limited resources on cloud
    connection = httplib.HTTPSConnection(CAT_API_SERVER)
    headers = {'content-type': 'application/json'}
    url = 'http://%s/api/v2.0/testrun/?api_key=%s&username=%s' % (
        CAT_API_SERVER, api_key, username)
    deliverable_endpoint = '/api/v2.0/deliverable'
    deliverables = []
    for id_ in deliverable_id:
        deliverables.append('%s/%s/' % (deliverable_endpoint, id_))
    pylogger.info('deliverable list %s' % deliverables)
    body = {"deliverables": deliverables,
            "workunit": "/api/v2.0/workunit/%s/" % workunit_id}
    body = json.dumps(body)
    connection.request(constants.HTTPVerb.POST, url, body, headers)
    response = connection.getresponse()

    status = response.status
    raw_data = response.read()
    if status != httplib.CREATED:
        pylogger.error("Retry testrun for workunit %s failed: %s" % (
            workunit_id, status))
        raise RuntimeError(raw_data)
    data = json.loads(raw_data)
    testrun_url = 'https://%s/testrun/%s' % (
        CAT_API_SERVER,  data['id'])
    pylogger.info("Testrun started %s" % testrun_url)

if __name__ == '__main__':
    # Parse commandline arguments.
    parser = argparse.ArgumentParser()
    parser.add_argument("--deliverable", dest="deliverable", action="append",
                        help="Deliverable ID")
    parser.add_argument("--workunit", dest="workunit", action="store",
                        help="Workunit ID")
    # To get apikey, click on the username on cat.eng.vmware.com and
    # follow instructions given there
    parser.add_argument("--apikey", dest="api_key", action="store",
                        help="API key for write access")
    parser.add_argument("--username", dest="username", action="store",
                        help="CAT user name")
    options = parser.parse_args()

    retry_testrun(
        workunit_id=options.workunit, deliverable_id=options.deliverable,
        api_key=options.api_key, username=options.username)


#! /usr/bin/env python
#coding=utf-8
from __future__ import unicode_literals
import sys,os
reload(sys)
sys.setdefaultencoding('utf-8')
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
sys.path.append("../")
from Whoosh_Module.Whoosh_Module import Whoosh_Module
from flask import Flask,url_for,render_template,request

Whoosh = None
flask_app = Flask(__name__)
@flask_app.route('/',methods=['GET', 'POST'])
def index():
    global Whoosh
    if request.method == 'POST':
        keyword=request.form['keyword']
        results = Whoosh.search(keyword)
        return render_template('result.html',results=results,keyword=keyword)
    else :
        return render_template('index.html',results=None)

class Flask_Module(object):
    def __init__(self,WhooshName):
        global Whoosh
        Whoosh = WhooshName
    def run(self):
        #flask_app.debug=True
        flask_app.run(host='0.0.0.0')

if __name__ == '__main__':
    Whoosh=Whoosh_Module()
    #flask_app.debug=True
    flask_app.run(host='0.0.0.0')