import threading
import time
import math
from jira import JIRA
from everisJIO import everisJIO
from fastUtils import fastUtils
from getSubtasks import getSubtasks
from workloadSQL import workloadSQL
import logging
from logger import logger

options = {
    'server': 'https://steps.everis.com/jiraesp2/'
    }

log = logger.getCustomLogger("workloadThreads")
log.debug(" -- Iniciando workloadThreads --")

numThreads = 48
allrunEsp = 'ALLRUN-1079'

cargaInicial = "project='Servicio RUN' and issuetype not in subTaskIssueTypes()"
lastDay = "project='Servicio RUN' and issuetype not in subTaskIssueTypes() and ( updated>-1d  or issuefunction in parentsOf('updated > -1d'))"
busqAllRUNEsp = "Project='Servicio RUN' and key='"+allrunEsp+"'"

print(" -- WORKLOADTHREADS START --")

jira = JIRA(options, auth=('mespinof', '!L1berate45'))

totalIssues = jira.search_issues(lastDay,startAt=0, maxResults=1).total
batchPerThread = int(math.ceil(totalIssues/numThreads))

print("Division: {}".format(batchPerThread))

allruns = []

def getIssues(startPoint, batchResults, thread):
    print("\n---- Se inicia el Thread{} ----\n".format(thread))
    print("\n---- Thread{}:\n---- startPoint:{}\n---- batchResults{}:".format(thread,startPoint,batchResults))
    #issues = jira.search_issues("project='Servicio RUN' and issuetype not in subTaskIssueTypes()",startAt=startPoint, maxResults=batchResults)
    issues = jira.search_issues(lastDay,startAt=startPoint, maxResults=batchResults, expand='changelog')
    for x in issues:
        allrun = everisJIO(x)
        allrun.worklog = jira.worklogs(allrun.id)
        count = 0
        for y in x.fields.subtasks:
            try:
                allrun.subtasks.append(getSubtasks.getSubtask(y.key, jira))
                allrun.subtasks[count].worklog = jira.worklogs(y.key)
            except AttributeError:
                print("Ha habido un problema recuperando el worklog de la tarea: {}".format(y.key))
            count+=1
        allruns.append(allrun)

threads_list = []
startPoint = 0
startTime = time.time()

print("\n\n ----------  INICIAMOS EXTRACCIÓN ---------- \n\n")
for i in range(numThreads):
    t = threading.Thread(target = getIssues,
                         name = 'thread{}'.format(i),
                         args = (startPoint,batchPerThread,i))
    threads_list.append(t)
    t.start()
    startPoint += batchPerThread

for t in threads_list:
    t.join()

endTime = time.time()
print("\n\n Allruns extraídas: {}  -- Tiempo transcurrido: {} \n\n".format(len(allruns),(endTime-startTime)))
print("\n\n ----------  FINALIZAMOS EXTRACCIÓN ---------- \n\n")

print("\n\n ----------  INSERTAMOS EN BBDD  -----------\n\n")
workloadSQL.insertRegisters(allruns)
endTime = time.time()
print("\n\n Allruns grabadas en BBDD: {}  -- Tiempo transcurrido: {} \n\n".format(len(allruns),(endTime-startTime)))
print("\n\n ----------  GRABADO EN BBDD  -----------\n\n")


# coding:utf-8

from bottle import route, post, request, redirect, jinja2_template as template
import app.models.Scheduler
model = app.models.Scheduler.Scheduler()

@route('/')
def index():
    result = model.load()
    return template('index', result = result)

@route('/new')
def new():
    return template('new')

@route('/edit/<id:int>')
def edit(id):
    return template('edit', i=model.edit(id))

@post('/done')
def done():
    post_data = {}
    post_data['name'] = request.forms.get('name')
    post_data['des'] = request.forms.get('des')
    post_data['id'] = request.forms.get('id')
    post_data['del'] = request.forms.get('del')
    model.done(post_data)

    redirect('/')


"""add original title

Revision ID: daf0305d8ed1
Revises: 29afac58afde
Create Date: 2020-10-16 11:56:34.109483

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'daf0305d8ed1'
down_revision = '29afac58afde'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('videos', sa.Column('original_name', sa.Text(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('videos', 'original_name')
    # ### end Alembic commands ###


import webapp2
from google.appengine.api import users
import jinja2
import os
import string

jinja_environment = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

class Error(webapp2.RequestHandler):
    def get(self):

        user = users.get_current_user()
        if user:

            #set stylesheets needed per page
            specific_urls = """
                <link type="text/css" rel="stylesheet" href="/stylesheets/""" + self.__class__.__name__ + """.css" />
            """

            #add the page query to the html
            url = self.request.url
            url = string.split(url, '/')
            Error_template_values = {
                'page': url[len(url) - 1]
            }

            template = jinja_environment.get_template('Page_Content/error.html')
            Error_template = template.render(Error_template_values)

            url = users.create_logout_url(self.request.uri)
            nav = """

Dashboard

Design

About

Logout

            """ % url

            template_values = {
                'specific_urls':specific_urls,
                'nav': nav,
                'content': Error_template
            }

            template = jinja_environment.get_template('index.html')
            self.response.out.write(template.render(template_values))
        else:
            self.redirect('/')

app = webapp2.WSGIApplication([('/.*', Error)], debug=True)


from django.core.exceptions import MultipleObjectsReturned
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404, redirect
from django.views.generic import ListView, DetailView
from django.http import JsonResponse

from .forms import PrintersListForm, PrinterDetailForm
from .models import CustomWebPage, Printer, PrintOperation, VoteUnit

def vote(request, printer_id):
    printer = get_object_or_404(Printer, pk=printer_id)

    plus = request.POST['choice']
    vote = VoteUnit()
    vote.printer_id = printer.id
    vote.rating = 1
    vote.save()

    # selected_choice.save()
    # Always return an HttpResponseRedirect after successfully dealing
    # with POST data. This prevents data from being posted twice if a
    # user hits the Back button.
    return HttpResponseRedirect('print:printer_detail', args=(printer.id,))

def Complaint (request, pk, problem):
    printer = Printer.objects.get(id=pk)

    if (request.user not in printer.users.all()):
        new = 1
        if (problem == '1'):
            printer.paper_jam = True
            printer.users.add(request.user)
            printer.save()
        elif (problem == '2'):
            printer.no_paper = True
            printer.users.add(request.user)
            printer.save()
        elif (problem == '3'):
            printer.no_paint = True
            printer.users.add(request.user)
            printer.save()
        elif (problem == '4'):
            printer.problem = True
            printer.users.add(request.user)
            printer.save()
        else:
            pass
    else:
        new = 0
        if (problem == '1'):
            printer.paper_jam = False
            printer.users.remove(request.user)
            printer.save()
        elif (problem == '2'):
            printer.no_paper = False
            printer.users.remove(request.user)
            printer.save()
        elif (problem == '3'):
            printer.no_paint = False
            printer.users.remove(request.user)
            printer.save()
        elif (problem == '4'):
            printer.problem = False
            printer.users.remove(request.user)
            printer.save()
        else:
            pass

    return JsonResponse({'new': new})

def custom_page(request, desirable_page_id):
    try:
        selected_page = get_object_or_404(CustomWebPage, page_id=desirable_page_id)
    except (KeyError, MultipleObjectsReturned):
        selected_page = CustomWebPage.objects.filter(page_id=desirable_page_id)[0]
    return render(request, 'print/custom_page.html', {'selected_page': selected_page})

class PrintersList(ListView):
    template_name = "print/printers_list.html"
    model = Printer

    def dispatch(self, request, *args, **kwargs):
        self.form = PrintersListForm(request.GET)
        self.form.is_valid()
        return super(PrintersList, self).dispatch(request, *args, **kwargs)

    def get_queryset(self):
        queryset = Printer.objects.all()
        if self.form.cleaned_data.get('search'):
            queryset = queryset.filter(id=self.form.cleaned_data['search'])
        if self.form.cleaned_data.get('active_only') is not None:
            is_active = self.form.cleaned_data['active_only']
            if is_active == 'true':
                queryset = queryset.filter(is_active=True)
            elif is_active == 'false':
                queryset = queryset.filter(is_active=False)
        if self.form.cleaned_data.get('sort_field'):
            queryset = queryset.order_by(self.form.cleaned_data['sort_field'])
        return queryset

    def get_context_data(self, **kwargs):
        context = super(PrintersList, self).get_context_data(**kwargs)
        context['form'] = self.form
        return context

class PrinterDetail(DetailView):
    template_name = 'print/printer_detail.html'
    model = Printer

    def dispatch(self, request, pk=None, *args, **kwargs):
        self.current_printer_id = pk
        self.form = PrinterDetailForm(request.POST)
        return super(PrinterDetail, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(PrinterDetail, self).get_context_data(**kwargs)
        context['form'] = self.form
        return context

    def post(self, request, *args, **kwargs):
        if self.form.is_valid():
            printer = Printer.objects.get(id=self.current_printer_id)
            if self.form.cleaned_data['enable_choice'] == 'true':
                printer.is_active = True
            elif self.form.cleaned_data['enable_choice'] == 'false':
                printer.is_active = False

            printer.save()
            return redirect("print:printer_detail", pk=self.current_printer_id)
            # raise forms.ValidationError('Выберите действие!')
            #return redirect("print:printer_detail", pk=self.current_printer_id)

class OperationsListAjax(ListView):
    template_name = "print/operations_ajax.html"
    model = PrintOperation

class PrintersListAjax(ListView):
    template_name = "print/printers_list_ajax.html"
    model = Printer

class PrinterStatusAjax(ListView):
    template_name = "print/printer_detail_ajax.html"
    model = Printer

class OperationDetail(DetailView):
    template_name = 'print/operation_detail.html'
    model = PrintOperation

class PrinterVote(DetailView):
    template_name = 'print/operation_detail.html'
    model = VoteUnit


import numpy as np
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from dataset.ModelNet40 import RegistrationData,ModelNet40
#from model.Pointnet import PointNet
from model.iPCRNet import iPCRNet
from model.MeshNet import MeshNet
from losses.chamfer_distance import ChamferDistanceLoss
import transforms3d
from losses.rmse_features import RMSEFeaturesLoss
from losses.frobenius_norm import FrobeniusNormLoss
from operations.transform_functions import PCRNetTransform

BATCH_SIZE=20
EVAL=False
START_EPOCH=0
MAX_EPOCHS=200
pretrained='checkpoints/ipcrnet/models2/best_model.t7'       #使用最好的模型参数测试

# Find error metrics.
def find_errors(igt_R, pred_R, igt_t, pred_t):
    # igt_R:				Rotation matrix [3, 3] (source = igt_R * template)
    # pred_R: 			Registration algorithm's rotation matrix [3, 3] (template = pred_R * source)
    # igt_t:				translation vector [1, 3] (source = template + igt_t)
    # pred_t: 			Registration algorithm's translation matrix [1, 3] (template = source + pred_t)

    # Euler distance between ground truth translation and predicted translation.
    igt_t = -np.matmul(igt_R.T, igt_t.T).T			# gt translation vector (source -> template)
    translation_error = np.sqrt(np.sum(np.square(igt_t - pred_t)))

    # Convert matrix remains to axis angle representation and report the angle as rotation error.
    error_mat = np.dot(igt_R, pred_R)							# matrix remains [3, 3]
    _, angle = transforms3d.axangles.mat2axangle(error_mat)
    return translation_error, abs(angle*(180/np.pi))

def compute_accuracy(igt_R, pred_R, igt_t, pred_t):
    errors_temp = []
    for igt_R_i, pred_R_i, igt_t_i, pred_t_i in zip(igt_R, pred_R, igt_t, pred_t):
        errors_temp.append(find_errors(igt_R_i, pred_R_i, igt_t_i, pred_t_i))
    return np.mean(errors_temp, axis=0)

def test_one_epoch(device, model, test_loader):
    model.eval()
    test_loss = 0.0
    count = 0
    errors = []

    for i, data in enumerate(tqdm(test_loader)):
        template, source, igt, igt_R, igt_t = data

        for index in range(5):
            template[index] = template[index].to(device)  # [B,N,3]
            source[index] = source[index].to(device)  # [B,N,3]

        # template = template.to(device)
        # source = source.to(device)
        igt = igt.to(device)

        # source_original = source.clone()
        # template_original = template.clone()
        igt_t = igt_t - torch.mean(source[4], dim=1).unsqueeze(1)
        # source[4] = source[4] - torch.mean(source[4], dim=1, keepdim=True)
        # template[4] = template[4] - torch.mean(template[4], dim=1, keepdim=True)
        for index in range(3):
            source[index] = source[index] - torch.mean(source[index], dim=2, keepdim=True)
            template[index] = template[index] - torch.mean(template[index], dim=2, keepdim=True)
        source[4] = source[4] - torch.mean(source[4], dim=1, keepdim=True)
        template[4] = template[4] - torch.mean(template[4], dim=1, keepdim=True)

        output = model(device, template, source)
        est_R = output['est_R']
        est_t = output['est_t']
        est_T = output['est_T']

        errors.append(compute_accuracy(igt_R.detach().cpu().numpy(), est_R.detach().cpu().numpy(),
                                       igt_t.detach().cpu().numpy(), est_t.detach().cpu().numpy()))

        # transformed_source = torch.bmm(est_R, source.permute(0, 2, 1)).permute(0,2,1) + est_t
        loss_val = ChamferDistanceLoss()(template[4], output['transformed_source'])

        test_loss += loss_val.item()
        count += 1

    test_loss = float(test_loss)/count
    errors = np.mean(np.array(errors), axis=0)
    return test_loss, errors[0], errors[1]

if __name__ == '__main__':
    testset = RegistrationData('PCRNet', ModelNet40(part='test'),is_testing=True)
    testloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, drop_last=False, num_workers=4)
    device = torch.device('cpu')

    #ptnet = PointNet(emb_dims=1024)
    model = iPCRNet(feature_model=MeshNet)
    model = model.to(device)

    if pretrained:
        model.load_state_dict(torch.load(pretrained, map_location='cpu'))

    model.to(device)
    test_loss, translation_error, rotation_error = test_one_epoch(device, model, testloader)
    print("Test Loss: {}, Rotation Error: {} & Translation Error: {}".format(test_loss, rotation_error,translation_error))


# -*- coding: utf-8 -*-

"""main module.

@route '/'
@route 'web_hook'
"""

import os
import requests
import json

from flask import Flask
from flask import request

from doco.client import Client

from models import db
from models import User
from models import Flag
from models import Reply
from modules.bot import Bot
from modules.message import Quiz
from modules.utils import GeneralUtility
from modules.utils import Config
from modules.utils import Header
from modules.webhook import ActionEvents
from modules.webhook import Events
from modules.yahoo_api import KeyPhrase

app = Flask(__name__)

# set database
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['DATABASE_URL']

db.init_app(app)

config = Config('./setting.conf')
ACCESS_TOKEN = config.get_access_token()
ENDPOINT_URL = config.get_endpoint_url()
APP_ID = config.get_appid()

PROXY = {
    "https": os.environ.get('FIXIE_URL', ''),
    "http": os.environ.get('FIXIE_URL', '')
}

@app.before_first_request
def create_database():
    db.create_all()

    if not db.session.query(Flag).filter(Flag.key == "reply_flag").count():
        reg = Flag("reply_flag")
        reg.set_value(0)
        db.session.add(reg)
        db.session.commit()

@app.route('/')
def index_route():
    return 'Running!'

@app.route('/web_hook', methods=['POST'])
def webhook_route():
    bot = Bot()

    # create header
    header = Header(ACCESS_TOKEN)
    content_header = header.content_header()

    # request event
    event = request.json['events'][0]

    if 'postback' in event:
        try:
            events = ActionEvents(event)
            reply_token = events.replyToken
        except:
            raise

        if events.postback.data == "o":
            content = bot.text_message_content(reply_token, "せいかいだよ！")
        else:
            content = bot.text_message_content(reply_token, "いいえ")

    else:
        try:
            events = Events(event)
            reply_token = events.replyToken
        except:
            raise

        userid = events.source.userId
        if not db.session.query(User).filter(User.userid == userid).count():
            reg = User(events.source.userId)
            db.session.add(reg)
            db.session.commit()

        # extract user
        user = User.query.filter(User.userid == userid).first()
        # extract reply flag
        reply_flag = Flag.query.filter(Flag.key == "reply_flag").first()

        reg = Reply(events.source.userId, events.message.text)

        try:
            keyphrase = KeyPhrase(APP_ID, events.message.text)
            keywords = keyphrase.keywords
        except:
            keywords = {}

        # search image
        utility = GeneralUtility()
        if len(keywords) != 0 and utility.get_match_key(keywords, "画像"):
            image_url = bot.get_image(' '.join(keywords.keys()))
            content = bot.image_message_content(reply_token, image_url)
        elif events.message.text == "なぞなぞ":
            quiz = Quiz("data/quiz.csv")
            quiz_text, actions = quiz.create_quiz()
            content = bot.buttons_message_content(
                reply_token,
                quiz_text,
                actions
            )
        elif reply_flag.value == 1:
            reply_flag.set_value(0)
            user.set_username(events.message.text)

            db.session.add(reply_flag)
            db.session.add(user)
            db.session.commit()

            content = bot.text_message_content(
                reply_token,
                "わかった！よろしくね、%s!" % user.username
            )
        elif user.username is None:
            reply_flag.set_value(1)

            db.session.add(reply_flag)
            db.session.commit()

            content = bot.text_message_content(
                reply_token,
                "あなたのお名前はなあに？"
            )
        else:
            docomo_client = Client(apikey=config.get_docomo_apikey())
            content = bot.text_message_content(
                reply_token,
                docomo_client.send(
                    utt=events.message.text,
                    apiname='Dialogue',
                    **{'nickname': user.username}
                )['utt']
            )

    request_body = content.get_request_body()

    # post request
    res = requests.post(
        ENDPOINT_URL,
        data=json.dumps(request_body),
        headers=content_header,
        proxies=PROXY
    )
    print(res)

    return 'Running'

if __name__ == '__main__':
    app.run()


from pathlib import Path

import pytest

from ogc import run
from ogc.enums import SpecCore
from ogc.exceptions import SpecProcessException
from ogc.spec import SpecJobMatrix, SpecJobPlan, SpecLoader, SpecPlugin
from ogc.state import app

fixtures_dir = Path(__file__).parent / "fixtures"

@pytest.fixture(scope="module")
def runners():
    """ Fixture with the parsed runners
    """
    spec = SpecLoader.load([fixtures_dir / "spec.yml"])
    return [job for job in spec["plan"]]

def test_matrix_combos(mocker):
    """ Test all combos are present
    """
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec-matrix.yml"])
    matrixes = SpecJobMatrix(spec[SpecCore.MATRIX])
    combos = matrixes.generate()
    assert {
        "snap_version": "1.18/edge",
        "series": "focal",
        "channel": "stable",
        "arch": "arm64",
    } in combos
    assert {
        "snap_version": "1.17/stable",
        "series": "bionic",
        "channel": "stable",
        "arch": "arm64",
    } in combos
    assert {
        "snap_version": "1.15/edge",
        "series": "xenial",
        "channel": "edge",
        "arch": "amd64",
    } in combos

def test_matrix_replace_env_var(mocker):
    """ Tests that a matrix variable updates an referenced environment variable
    """
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec-matrix.yml"])
    matrixes = SpecJobMatrix(spec[SpecCore.MATRIX])
    jobs = [SpecJobPlan(spec[SpecCore.PLAN], matrix) for matrix in matrixes.generate()]
    jobs[0].env()
    assert app.env["JUJU_CONTROLLER"] == "validate-ck-{}".format(
        jobs[0].matrix["series"]
    )
    assert app.env["JUJU_DEPLOY_CHANNEL"] == jobs[0].matrix["channel"]

def test_yml_include_spec(mocker):
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec-base.yml"])
    assert "juju" in spec["plan"][0]["before-script"][0]

def test_nested_assets(mocker):
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec.yml"])
    plug = SpecPlugin(spec[SpecCore.PLAN][0]["script"][3]["runner"])
    assets = plug.opt("assets")
    assert assets[0]["name"] == "pytest configuration"

def test_cmd_to_env(mocker):
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec.yml"])
    _env = {}
    _env["BONZAI"] = "bonzai-test"
    _env["ANOTHERTIME"] = "this happened late"
    _env["VAR_NICE"] = "interesting concept"
    _env["CONTROLLER"] = "juju-controller"
    _env["MODEL"] = "juju-model"
    app.env = _env
    spec = SpecPlugin(spec[SpecCore.PLAN][0]["script"][0]["runner"])
    cmd = spec.opt("cmd")
    assert cmd == (
        "echo bonzai-test lthis happened "
        "late envinteresting concept "
        "juju-controller:juju-model "
        "juju-controller juju-model"
    )

def test_get_option_env_key(mocker):
    """ Tests that an environment variable set for a given option is converted
    into the hosts environment setting """
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec.yml"])
    plug = SpecPlugin(spec[SpecCore.PLAN][0]["before-script"][0]["juju"])

    _env = {}
    _env["JUJU_CLOUD"] = "aws/us-east-1"
    _env["JUJU_CONTROLLER"] = "test-controller"
    _env["JUJU_MODEL"] = "test-model"
    app.env = _env
    assert plug.opt("cloud") == "aws/us-east-1"
    assert plug.opt("controller") == "test-controller"
    assert plug.opt("model") == "test-model"

def test_get_option_env_key_bool(mocker):
    """ Tests that get_plugin_option handles boolean values correctly
    """
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec.yml"])
    plug = SpecPlugin(spec[SpecCore.PLAN][0]["before-script"][0]["juju"])

    _env = {}
    _env["JUJU_CLOUD"] = "aws/us-east-1"
    _env["JUJU_CONTROLLER"] = "test-controller"
    _env["JUJU_MODEL"] = "test-model"
    app.env = _env
    assert plug.opt("deploy.reuse") is True

def test_run_script_passes_check(mocker):
    """ Tests that we can run shell commands
    """
    mocker.patch("ogc.state.app.log")
    run.script("ls -l", env=app.env.copy())

def test_run_script_blob_passes_check(mocker):
    """ Tests that we can run shell scripts
    """
    mocker.patch("ogc.state.app.log")
    blob = """
#!/bin/bash
set -x
ls -l
"""
    run.script(blob, env=app.env.copy())

def test_run_script_fails_check(mocker):
    """ Tests that we can run shell scripts
    """
    mocker.patch("ogc.state.app.log")
    with pytest.raises(SpecProcessException):
        run.script("ls -l\necho HI\nexit 1", env=app.env.copy())

@pytest.mark.skip
def test_condition_if(mocker):
    """ Tests that a condition will skip a job item
    """
    mocker.patch("ogc.state.app.log")
    spec = SpecLoader.load([fixtures_dir / "spec-condition.yml"])
    matrixes = [SpecJobMatrix(matrix) for matrix in app.spec[SpecCore.MATRIX]]

    jobs = [SpecJobPlan(app.spec[SpecCore.PLAN], matrix) for matrix in matrixes]
    assert jobs[0].condition_if()
    assert jobs[1].condition_if()
    assert jobs[2].condition_if() is False


# -*- coding: utf-8 -*-
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import padding
import base64
import hashlib

def md5Hex(instr):
    return hashlib.md5(instr.encode(encoding='UTF-8')).hexdigest()

def md5HexJy(instr, keystr):
    return md5Hex(md5Hex(md5Hex(instr))+keystr)

class aes(object):
    def __init__(self,key,iv):
        self.key = key
        self.iv = iv
        key_b = bytes(self.key, 'utf-8')
        iv_b = bytes(self.iv, 'utf-8')
        self.cipher = Cipher(algorithms.AES(key_b), modes.CBC(iv_b), backend=default_backend())

    def encodeJson(self,line):
        byline = bytes(line, 'utf-8')
        encryptor = self.cipher.encryptor()
        padder = padding.PKCS7(128).padder()
        padded_data = padder.update(byline) + padder.finalize()
        enc_content = encryptor.update(padded_data) + encryptor.finalize()
        stren =base64.b64encode(enc_content)
        return stren.decode()

    def decodeJson(self,enStr):
        deStr = bytes(enStr,'utf-8')
        destrendd = base64.b64decode(deStr)
        depadder = padding.PKCS7(128).unpadder()
        decryptor = self.cipher.decryptor()
        padded_data = decryptor.update(destrendd) + decryptor.finalize()
        dec_content = depadder.update(padded_data) + depadder.finalize()
        return dec_content.decode()


import pyautogui, time #imports pyautogui and time modules
from time import sleep #imports sleep from time module

##########
count = 0#  #counter
##########

while True:
    time.sleep(10) #you can change the time if u want (second)
    count += 1 #Don't change this
    myScreenshot = pyautogui.screenshot() #Takes the screenshot
    myScreenshot.save(r'path here\screenshot{}.png'.format(count)) #saves the screenshot, change "path here".


import discord
from discord.ext import commands

client = commands.Bot(command_prefix = '.')

@client.event
async def on_ready():
    print('Hello World!')

####### METHODS #######

#Watch list
#Adds users to the specified group for mentioning later
@client.command(aliases=['bought'])
async def watch(ctx):
    role = discord.utils.get(ctx.guild.roles, name="Stalnks")
    user = ctx.message.author
    userMessage = ctx.message
    botMessage = user.mention + ', here\'s hoping for a nice surprise!'
    await user.add_roles(role)
    botMessageEvent = await ctx.send(botMessage)
    await userMessage.delete(delay=1)
    await botMessageEvent.delete(delay=5)

#Unwatch
#Removes users from the specified group
@client.command(aliases=['sold'])
async def unwatch(ctx):
    role = discord.utils.get(ctx.guild.roles, name="Stalnks")
    user = ctx.message.author
    userMessage = ctx.message
    botMessage = user.mention + ', pleasure doin\' business with ya!'
    await user.remove_roles(role)
    botMessageEvent = await ctx.send(botMessage)
    await userMessage.delete(delay=1)
    await botMessagEvent.delete(delay=5)

#buying
#Mentions the role with the price and the dodo code for travelling
#PARAMETERS:
    # bellAmount: Price at Nook's Cranny on the island
    # dodoCode: Dodo Code from the Airport for travelling

@client.command()
async def buying(ctx, bellAmount, dodoCode: str):
    authorUser = ctx.message.author
    mentionRole = discord.utils.get(ctx.guild.roles, name="Stalnks")
    openRole = discord.utils.get(ctx.guild.roles, name="Open Town")
    await authorUser.add_roles(openRole)
    if(bellAmount and dodoCode):
        await ctx.send(mentionRole.mention + ' ' + authorUser.mention + '\'s island is buying turnips for ' + str(bellAmount) + ' bells!' +
                    '\n Their Dodo Code is: ' + dodoCode)

#Stop selling
#Edits the previous message to say that the island is closed
@client.command(aliases=['stop'])
async def stop_selling(ctx):
    channel = ctx.message.channel
    authorUser = ctx.message.author
    openRole = discord.utils.get(ctx.guild.roles, name="Open Town")
    #I have a feeling once a channel has a lot of messages this will become unwieldy
    async for m_id in channel.history().filter(lambda m: authorUser in m.mentions).filter(lambda m: 'island is' in m.content).map(lambda m:m):
        messageToEdit = m_id
        await authorUser.remove_roles(openRole)
        await messageToEdit.edit(content=authorUser.mention + ' has closed their town.')
        await messageToEdit.delete(delay=3600)
    #I have a feeling once a channel has a lot of messages this will become unwieldy
    async for m_id in channel.history().filter(lambda m: m.author == authorUser).filter(lambda m: '.selling' in m.content).map(lambda m:m):
        botSummonMessage = m_id
        await botSummonMessage.delete(delay=3)
    #I have a feeling once a channel has a lot of messages this will become unwieldy
    async for m_id in channel.history().filter(lambda m: m.author == authorUser).filter(lambda m: '.buying' in m.content).map(lambda m:m):
        botSummonMessage = m_id
        await botSummonMessage.delete(delay=3)
    await ctx.message.delete(delay=3)

#Set Fruit role
#Set the role in the server based on your Fruit
#PARAMETERS:
    # Fruit: name of the fruit to get role
@client.command()
async def fruit(ctx, fruit: str):
    fruit = fruit.lower()
    authorUser = ctx.message.author
    peachRole = discord.utils.get(ctx.guild.roles, name="just peachy")
    orangeRole = discord.utils.get(ctx.guild.roles, name="orange you glad")
    appleRole = discord.utils.get(ctx.guild.roles, name="apple a day...")
    pearRole = discord.utils.get(ctx.guild.roles, name="thank you pear-y much")
    cherryRole = discord.utils.get(ctx.guild.roles,name="so cherry sweet")

    if(peachRole in authorUser.roles or orangeRole in authorUser.roles or appleRole in authorUser.roles or
            pearRole in authorUser.roles or cherryRole in authorUser.roles):
        await ctx.send("You already have a fruit!")

    else:
        if(fruit == 'cherry'):
            await authorUser.add_roles(cherryRole)
            await ctx.send("You have set your fruit to Cherry!")
        elif(fruit == 'orange'):
            await authorUser.add_roles(orangeRole)
            await ctx.send("You have set your fruit to Orange!")
        elif(fruit == 'peach'):
            await authorUser.add_roles(peachRole)
            await ctx.send("You have set your fruit to Peach!")
        elif(fruit == 'apple'):
            await authorUser.add_roles(appleRole)
            await ctx.send("You have set your fruit to Apple!")
        elif(fruit == 'pear'):
            await authorUser.add_roles(pearRole)
            await ctx.send("You have set your fruit to Pear!")

#Help
@client.command()
async def dmhelp(ctx):
    await ctx.send("Why hullo there! I'm Daisy Mae!")
    await ctx.send("Need some help?")

    await ctx.send("Prefix: '.'\nAdd Fruit: **.fruit** [*cherry*,*orange*,*peach*,*apple*,*pear*]\n" +
    "\nAdd to Watchlist: **.bought**/**.watch**\n\nRemove from watchlist: **.sold**/**.unwatch**\n"+
    "\nAnnounce buying & being open: **.buying**\n\nClose town (use after .buying): **.stop**")

client.run('')


import functools

from flask import render_template, current_app, session as flask_session
from ..db import Player

def redirect_page(title, message, redirect_url, redirect_delay=0):
    message = message.format(delay=redirect_delay)
    return render_template('redirect.html', title=title, message=message,
                           url=redirect_url, delay=redirect_delay)

def inject_current_player(func):
    @functools.wraps(func)
    def wrapped(*args, **kwargs):
        with current_app.db.session_scope() as session:
            if 'username' in flask_session:
                current_player = session.query(Player). \
                    filter_by(name=flask_session['username']).one_or_none()
            else:
                current_player = None
            return func(session, current_player, *args, **kwargs)
    return wrapped


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

dataset = pd.read_csv('train.csv')

X_train = dataset.iloc[:, 1:13].values
Y_train = dataset.iloc[:,[13]].values
df_y_train= pd.DataFrame(Y_train)
df_x_train  = pd.DataFrame(X_train)

### binarize Y train column
Y_train =np.where(Y_train == 'correct',1,0)

## handling missing values
from sklearn.preprocessing import Imputer
imputer_mode =Imputer(missing_values ='NaN', strategy ='most_frequent',axis =0)
imputer_mode.fit(X_train[:,[0]])
X_train[:,[0]]=imputer_mode.transform(X_train[:,[0]])

imputer = Imputer(missing_values ='NaN', strategy ='most_frequent',axis =0)
imputer.fit(X_train[:,[1,2,3,4,11]])
X_train[:,[1,2,3,4,11]] = imputer.transform(X_train[:,[1,2,3,4,11]])
df = pd.DataFrame(X_train)

xxc=pd.DataFrame(X_train)
xxc[5]=pd.to_datetime(xxc[5])
xxc[6]=pd.to_datetime(xxc[6])
xxc[5]=xxc[5].map(dt.datetime.toordinal)
xxc[6]=xxc[6].map(dt.datetime.toordinal)
X_train[:,5]=xxc[5]
X_train[:,6]=xxc[6]

##DO the same for Test set
test_dataset =pd.read_csv('test.csv')

X_test = test_dataset.iloc[:, 1:13].values
df_x_test  = pd.DataFrame(X_test)

xxc=pd.DataFrame(X_test)
xxc[5]=pd.to_datetime(xxc[5])
xxc[6]=pd.to_datetime(xxc[6])
xxc[5]=xxc[5].map(dt.datetime.toordinal)
xxc[6]=xxc[6].map(dt.datetime.toordinal)
X_test[:,5]=xxc[5]
X_test[:,6]=xxc[6]

Test_ids = test_dataset.iloc[:,0:1]

imputer_mode =Imputer(missing_values ='NaN', strategy ='most_frequent',axis =0)
imputer_mode.fit(X_test[:,[0]])
X_test[:,[0]]=imputer_mode.transform(X_test[:,[0]])

imputer = Imputer(missing_values ='NaN', strategy ='most_frequent',axis =0)
imputer.fit(X_test[:,[1,2,3,4,11]])
X_test[:,[1,2,3,4,11]] = imputer.transform(X_test[:,[1,2,3,4,11]])

## Now back to work................

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train[:,[0,1,2,3,4,5,6,7,8,9,10,11]])
X_test = scaler.fit_transform(X_test[:,[0,1,2,3,4,5,6,7,8,9,10,11]])

from sklearn.linear_model import LogisticRegression

classifier = LogisticRegression(random_state=0)
classifier.fit(X_train,Y_train)

#Import test values

y_preds = classifier.predict(X_test)
Y_pred_dataframe = pd.DataFrame(y_preds)
Y_test_ids_dataframe = pd.DataFrame(Test_ids)

submission =pd.concat([Y_test_ids_dataframe,Y_pred_dataframe],axis=1)
submission =submission.rename(columns={"tiripid":"tripid"})
submission =submission.rename(columns={0:"prediction"})

submission.to_csv('submission.csv',index=False)


import pytest
import arrays_5.sample_data as s

def test_sample_data():
    subset_length = 4
    input_set = [5, 10, 20, 4, 3, 5]
    result = s.sample_data(input_set, subset_length)

    if len(result) != subset_length:
        pytest.fail(f'sample_data() result length is {len(result)}, while expected {subset_length}')


import logging
import random

import matplotlib.pyplot as plt
import numpy as np
import sys

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT, level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.propagate = True

class SGD_SVM(object):
    n = 0

    def __init__(self):
        self.w = None
        np.set_printoptions(precision=3)

    def calc_w(self, x, y, g=0.1, l=0.00001):
        return self._calc_w(self.w, x, y, g, l)

    def _calc_w(self, w, x_, y_, g=0.1, l=0.00001):
        v = y_ * np.dot(w, x_)
        return w - (g * l) * w + (0 if v > 1 else y_ * x_)

    def Qsvm(self, x_, y_, l=0.00001):
        return self._Qsvm(self.w, x_, y_, l)

    def _Qsvm(self, w, x_, y_, l=0.0001):
        v = y_ * np.dot(w, x_)
        loss = l * np.linalg.norm(w)**2 + max(0, 1 - v)
        grad = 0 if v > 1 else -y_ * x_
        return loss, grad

    def fw(self, x):
        return self._fw(self.w, x)[0]

    def _fw(self, w, x):
        value = np.dot(x, w)
        return np.sign(value), value

    def score(self, matrix, categories):
        n = 0; m = 0; v = []; s = []
        for x_, y_ in zip(matrix, categories):
            l, v_ = self._fw(self.w, x_)
            v.append(v_), s.append(y_)
            if l == y_:
                m += 1
            else:
                pass
                # logger.info("{} invece era {}".format(l, y_))
            n += 1

        return m / float(n), s, v

    def fit(self, matrix, categories):
        i = random.choice(range(matrix.shape[0]))
        self.w = self.calculate_w(matrix, categories, matrix[i].copy())
        return self.w

    def calculate_empirical_risk(self, x, y, w, g, l):
        loss = 0.0; n = len(y)
        for x_, y_ in zip(x, y):
            loss += l * np.linalg.norm(w) ** 2 + max(0, 1 - y_ * np.dot(w, x_))
        loss /= n
        return loss

    def calculate_w(self, x, y, w):

        l = 0.001
        g0 = 0.5; g = 0.0
        n = len(y); t = 1.0; i = 0

        old_p = int(100 * (t / n)); p = 0.9
        delta = np.inf; loss0 = np.inf; loss = 0

        # logger.info("{:>20}{:>20}{:>20}{:>20}".format("LOSS", "DELTA", "ACTUAL LOSS", "DELTA 2"))
        data = []
        while t < n:  # loss0 == np.inf or delta > 0.000001 or t < n:
            i = random.choice(range(n))
            x_, y_ = x[i], y[i]

            if (old_p != p and p % 10 == 0):
                logger.info("Completamento {}%".format(p))
                old_p = p
                #actual_q = self._Qsvm(w, x_, y_, l)[0]
                #loss = self.calculate_empirical_risk(x, y, w, g, l)
                #delta2 = np.abs(actual_q - loss)
                #delta = np.abs(loss0 - loss)
                #loss0 = loss
                #data.append( (loss, delta, actual_q, delta2) )
            p = int(100 * (t / n))

            g = g0 / (1 + l * g0 * t)
            # g = 1 / t
            grad = 0 if (y_ * np.dot(w, x_) > 1) else y_ * x_
            w = w - (g * l) * w + g * grad
            t += 1

            # v = y_ * np.dot(w, x_)
            # actual_q = self._Qsvm(w, x_, y_, l)[0]
            # loss = self.calculate_empirical_risk(x, y, w, g, l)
            # delta = np.abs(loss0 - loss)
            # delta2 = np.abs(actual_q - loss)
            # loss0 = loss
            # logger.info("{:>20}{:>20}{:>20}{:>20}".format(loss, delta, actual_q, delta2))

        #loss = self.calculate_empirical_risk(x, y, w, g, l)
        #delta2 = np.abs(actual_q - loss)
        #delta = np.abs(loss0 - loss)
        #logger.info("{:>20}{:>20}{:>20}{:>20}".format("LOSS", "DELTA", "ACTUAL LOSS", "DELTA 2"))
        #for d in data:
        #    logger.info("{:>20}{:>20}{:>20}{:>20}".format(d[0], d[1], d[2], d[3]))
        return w

if __name__ == '__main__':
    pass


import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
from sklearn.externals import joblib
import argparse
import os
# 加上48小时前obs 信息
# 处理 RAIN 值 去除 35以上数值
target_list=['t2m','rh2m','w10m']
from datetime import timedelta
from datetime import datetime
def datelist(beginDate, endDate):
    date_l=[datetime.strftime(x,'%Y-%m-%d') for x in list(pd.date_range(start=beginDate, end=endDate))]
    return date_l

if __name__=='__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--log-level', dest='log_level', default='info', type=str,
                        help='Logging level.')
    parser.add_argument('--model_dir', dest='model_dir',
                        default='../checkpoints/lgb',
                        type=str)
    parser.add_argument('--data_dir', dest='data_dir',
                        default='../data/testb7/merge', type=str)

    parser.add_argument('--dst_dir', dest='dst_dir',
                        default='../result/lgb')
    parser.add_argument('--first_day', dest='first_day',
                        default='20181028', type=str)
    parser.add_argument('--last_day', dest='last_day',
                        default='20181103', type=str)

    opt = parser.parse_args()

    feature_columns = ['t2m_obs', 'rh2m_obs', 'w10m_obs', 'psur_obs', 'q2m_obs', 'u10m_obs',
                       'v10m_obs', 'RAIN_obs',
                       't2m_prophet', 'rh2m_prophet', 'w10m_prophet',
                       't2m_M', 'rh2m_M', 'w10m_M', 'hour_sin', 'hour_cos', 'month_sin', 'month_cos',
                       'psfc_M', 'q2m_M', 'u10m_M', 'v10m_M',
                       'SWD_M', 'GLW_M', 'HFX_M', 'RAIN_M', 'PBLH_M', 'TC975_M', 'TC925_M',
                       'TC850_M', 'TC700_M', 'TC500_M', 'wspd925_M', 'wspd850_M', 'wspd700_M', 'wspd500_M']
    if opt.model_dir.endswith('_q'):
        feature_columns = feature_columns + ['Q975_M', 'Q925_M', 'Q850_M', 'Q700_M', 'Q500_M', 'LH_M']

    history_num = 24

    begin_dates = datelist(pd.to_datetime(opt.first_day) - timedelta(days=2),
                           pd.to_datetime(opt.last_day) - timedelta(days=2))
    dst_dates = datelist(opt.first_day, opt.last_day)
    end_dates = datelist(pd.to_datetime(opt.first_day) + timedelta(days=1),
                         pd.to_datetime(opt.last_day) + timedelta(days=1))

    model_dir = opt.model_dir
    data_dir = opt.data_dir
    dst_dir = opt.dst_dir
    if not os.path.exists(dst_dir):
        os.mkdir(dst_dir)

    for begin_date,dst_date,end_date in zip(begin_dates,dst_dates,end_dates):
        end_date=end_date+' 12-00-00'
        whole_submit = None
        for i in range(90001, 90011):
            index_list = []
            for j in range(37):
                index_list.append(str(i) + '_' + '{:02d}'.format(j))
            index = pd.DataFrame(index_list, columns=['FORE_data'])

            results = []
            for feature_index in range(3):

                lgb_model = joblib.load(os.path.join(model_dir,'model_' + str(i) + '_feature_' + str(feature_index) + '.m'))

                data_pre = pd.read_csv(os.path.join(data_dir,'merged_' + str(i) + '.csv'), index_col=0)

                data_pre.index = pd.to_datetime(data_pre.index)
                data_pre['hour'] = data_pre.index.hour
                data_pre['month'] = data_pre.index.month
                hour_period = 24 / (2 * np.pi)
                data_pre['hour_cos'] = np.cos(data_pre.index.hour / hour_period)
                data_pre['hour_sin'] = np.sin(data_pre.index.hour / hour_period)

                month_period = 12 / (2 * np.pi)
                data_pre['month_cos'] = np.cos(data_pre.index.month / month_period)
                data_pre['month_sin'] = np.sin(data_pre.index.month / month_period)

                data_pre['u10m_obs']=data_pre['u10m_obs']/data_pre['w10m_obs']
                data_pre['v10m_obs']=data_pre['v10m_obs']/data_pre['w10m_obs']
                data_pre['u10m_M']=data_pre['u10m_M']/data_pre['w10m_M']
                data_pre['v10m_M']=data_pre['v10m_M']/data_pre['w10m_M']

                data_pre = pd.DataFrame(data_pre, columns=feature_columns)
                data_pre = data_pre[begin_date:end_date]
                for col in data_pre.columns:
                    data_pre[col] = data_pre[col].fillna(data_pre[col].mean())

                pre_data_float = np.array(data_pre)
                # print(pre_data_float.shape)
                history = list(pre_data_float[24:48, feature_index])
                result = []
                for k in range(37):
                    row_data = pre_data_float[48 + k:48 + k + 1, 8:]
                    curr_history = np.array(history)[k:k + history_num]
                    obs_48h_ago = pre_data_float[k:k + 1, :8]
                    # print(row_data.shape)
                    # print(curr_history.shape)
                    curr_history = curr_history.reshape((1, -1))
                    X = np.c_[row_data, curr_history, obs_48h_ago]
                    #print(X.shape)
                    y = lgb_model.predict(X)
                    # print(y.shape)
                    if k < 4:
                        result.append(pre_data_float[48 + k, feature_index])
                        history.append(pre_data_float[48 + k, feature_index])
                    else:
                        result.append(y[0])
                        history.append(y[0])
                # print('result:', result)
                rmse = np.sqrt(mean_squared_error(pre_data_float[48:, feature_index], np.array(result)))
                rmse_M = np.sqrt(
                    mean_squared_error(pre_data_float[48:, feature_index], pre_data_float[48:, feature_index +11]))
                results.append(result)

                print('rmse:', rmse)
                print('rmse_M:', rmse_M)
                print('score:', (rmse_M - rmse) / rmse_M)
            suffix=opt.model_dir.split('/')[-1]
            submit = pd.DataFrame(np.array(results).T, columns=['t2m_'+suffix, 'rh2m_'+suffix, 'w10m_'+suffix])
            submit = pd.concat([index, submit], axis=1)
            if whole_submit is None:
                whole_submit = submit
            else:
                whole_submit = pd.concat([whole_submit, submit], axis=0)

        whole_submit.to_csv(os.path.join(dst_dir,dst_date + '.csv'), index=False)


from snownlp import SnowNLP
import jieba
import jieba.analyse
import sqlite3
import re
jieba.load_userdict("mydict.txt")

conn = sqlite3.connect(r'C:\Users\I321338\PycharmProjects\Trendar\db.sqlite3')
cursor = conn.cursor()

f = open(r"C:\Users\I321338\PycharmProjects\Trendar\machine_learning\test500.txt",'r', encoding='utf-8')

text = f.readline()
sentiment = [0,0,0,0,0,0,0,0,0,0]
while text!="":
    ###############################
    #如果要把句子根据标点符号分开再来判断，把下面三句话解开注释
    #还要把Snownlp(text)的text改成i，r.write(text)的text改成i
    ###############################

    # file = re.split("\n|，|。|！|\ " ,text)
    # for i in file:
    #      if i != '':
            text_snow = SnowNLP(text)
            score = text_snow.sentiments
            sentiment[int(score*10)] = sentiment[int(score*10)]+1
            text = f.readline()

print(sentiment)
cursor.execute("delete from dashboard_sentiment")
cursor.execute("update sqlite_sequence SET seq = 0 where name ='dashboard_sentiment'")
for num in sentiment:
    # num = int(n)
    # print(num)
    cursor.execute(" insert into dashboard_sentiment (number) values(?)",(num,))
conn.commit()


# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from .layers import LayerOutput, mixed_layer, identity_projection, \
    slope_intercept_layer
from .attrs import is_compatible_with
from .default_decorators import *
import activations as act

__all__ = []

def register_unary_math_op(op_name, act):
    def op(input, name=None):
        return mixed_layer(input=[identity_projection(input=input)],
                           name=name,
                           act=act)
    op = wrap_name_default(op_name)(op)
    op.__doc__ = type(act).__doc__
    globals()[op_name] = op
    __all__.append(op_name)

register_unary_math_op('exp', act.ExpActivation())
register_unary_math_op('log', act.LogActivation())
register_unary_math_op('abs', act.AbsActivation())
register_unary_math_op('sigmoid', act.SigmoidActivation())
register_unary_math_op('tanh', act.TanhActivation())
register_unary_math_op('square', act.SquareActivation())

def add(layeroutput, other):
    if is_compatible_with(other, float):
        return slope_intercept_layer(input=layeroutput, intercept=other)
    assert isinstance(other, LayerOutput)
    return mixed_layer(input=[identity_projection(input=layeroutput),
                              identity_projection(input=other)])

LayerOutput.__radd__ = add
LayerOutput.__add__ = add

def sub(layeroutput, other):
    if is_compatible_with(other, float):
        return slope_intercept_layer(input=layeroutput, intercept=other)
    assert isinstance(other, LayerOutput)
    neg = slope_intercept_layer(input=other, slope=-1.0)
    return mixed_layer(input=[identity_projection(input=layeroutput),
                              identity_projection(input=neg)])

LayerOutput.__sub__ = sub

def rsub(layeroutput, other):
    neg = slope_intercept_layer(input=layeroutput, slope=-1.0)
    return add(neg, other)

LayerOutput.__rsub__ = rsub


from django.db import models
from code_one.models import BankUser
import datetime

# Create your models here.
class DebitCard(models.Model):
    card_number = models.CharField(max_length=16, unique=True)
    date_created = models.DateTimeField(default=datetime.datetime.now())
    expiration_date = models.DateTimeField()
    bankUser = models.ForeignKey(BankUser)
    holder_first = models.CharField(max_length=20)
    holder_mid = models.CharField(max_length=1)
    holder_last = models.CharField(max_length=20)

    def __str__(self):
        return self.card_number

class Account(models.Model):
    balance = models.IntegerField(default=0)
    name = models.CharField(max_length=20)
    created_at = models.DateTimeField(default=datetime.datetime.now())
    last_deposit = models.DateTimeField(null=True, blank=True)
    last_withdraw = models.DateTimeField(null=True, blank=True)
    bankUser = models.ForeignKey(BankUser)
    debit_card = models.OneToOneField(DebitCard, on_delete=models.CASCADE, blank=True)

    def __str__(self):
        return self.name


import sys
from collections import defaultdict

my_dict = defaultdict(lambda:0)
my_dict["eric"] = 33
print(my_dict["eric"])
print(my_dict["fred"])
for foo, bar in sorted(my_dict.items()):
	print("%S --> %r % (foo, bar)")

print('hello world')

my_int = 4
my_float = 2.5
my_string = "hello"

print("string: %s\tfloat: %f\tint: %d" % (my_string, my_float, my_int))

my_variable = 5
if my_variable == 4:
	print("my_variable is 4")
else:
	print("my_variable is not 4")
for i in range(1, my_variable):
	print("i ==  %d" % (i))

sentence = "this is a pen"
words = sentence.split(" ")
for word in words:

	print(word)

print(" ||| ".join(words))

import sys
my_file = open(sys.argv[1], "r")
for line in my_file:
	line = line.strip()
if len(line) !=0:
	print(line)

my_file = open(sys.argv[1], "r")

for line in my_file:
	line = line.strip()
	if len(line) != 0:
		print(line)


import logging
import traceback

from aiohttp import web
from aiohttp.web_exceptions import HTTPError

import settings
from core.utils import json_to_str

log = logging.getLogger(__name__)

def _form_error_response(
    user_message: str, status_code: int, internal_message: str = None
):
    """Функция для формирования ответа с ошибками

    :param user_message:
    :param status_code:
    :param internal_message:
    :return:
    """
    body = {
        "errors": [
            {
                "message": user_message,
                "internal_message": internal_message,
            }
        ]
    }
    return web.Response(
        body=json_to_str(body).encode("utf-8"),
        content_type="application/json",
        status=status_code
    )

async def request_middleware(app, handler):
    """Прослойка для отлова базовых ошибок

    :param app:
    :param handler:
    :return:
    """
    async def middleware(request):
        try:
            response = await handler(request)
        except HTTPError as e:
            response = _form_error_response(
                user_message=e.reason,
                status_code=e.status_code
            )
        except:
            log.exception("Unexpected exception")
            response = _form_error_response(
                user_message="Sorry, the internal server error",
                internal_message=traceback.format_exc(),
                status_code=500
            )

        log.info(f"{request.method} {request.path}{request.query_string}")
        response.headers.add("Access-Control-Allow-Origin", settings.ACCESS_CONTROL_ALLOW_ORIGIN)
        response.headers.add("Access-Control-Allow-Headers", settings.ACCESS_CONTROL_ALLOW_HEADERS)
        return response

    return middleware


from django.core.urlresolvers import reverse_lazy
from django.contrib.auth.models import User
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.views import generic

from .models import Article
from .forms import ArticleForm

class BlogHomeListView(generic.ListView):
    template_name = "blog/article_home.html"
    model = Article

def article_detail(request, pk):
    article = Article.objects.get(id=pk)
    return render(request, "blog/article.html", context={'article': article})

def article_create(request):
    if request.method == "POST":
        form = ArticleForm(request.POST, request.FILES)

        if form.is_valid():
            try:
                curr_user = User.objects.get(username=request.user.username)
            except User.DoesNotExist:
                curr_user = User.objects.get(username='admin')

            instance = form.save(commit=False)
            instance.author = curr_user
            instance.save()
            return HttpResponseRedirect(form.instance.get_absolute_url())
    else:
        form = ArticleForm()

    return render(request, "blog/article_create.html", context={'form': form})

def article_delete(request, pk):
    success_url = reverse_lazy('blog:blog-home')
    article = Article.objects.get(id=pk)

    if request.method == "POST" and "delete" in request.POST:
        article.delete()
        return HttpResponseRedirect(success_url)
    else:
        return HttpResponseRedirect(article.get_absolute_url())

def article_update(request, pk):
    article = Article.objects.get(id=pk)

    if request.method == "POST":
        form = ArticleForm(request.POST, request.FILES, instance=article)

        if form.is_valid():
            try:
                user = User.objects.get(username=request.user.username)
            except User.DoesNotExist:
                user = User.objects.get(username='admin')
            instance = form.save(commit=False)
            instance.author = user
            instance.save()
            return HttpResponseRedirect(instance.get_absolute_url())
    else:
        form = ArticleForm(instance=article)

    return render(request, "blog/article_update.html", context={'form': form})


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('featured', '0003_auto_20150521_1436'),
    ]

    operations = [
        migrations.CreateModel(
            name='AssociatedEvent',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('featured_type', models.CharField(max_length=50, verbose_name=b'featured type')),
                ('featured_url', models.CharField(max_length=50, verbose_name=b'featured url')),
                ('plugin', models.ForeignKey(related_name='featured_event', to='featured.FeaturedPluginModel')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.RemoveField(
            model_name='featuredpluginmodel',
            name='featured_type',
        ),
        migrations.RemoveField(
            model_name='featuredpluginmodel',
            name='featured_url',
        ),
    ]


import win32com.client as win32
import os
import re
import pythoncom
import sys
from collections import Counter
from datetime import datetime
iter=sys.argv[1]
start=datetime.now()
print("--------------------------------------------------------------------------------------------------------")
print("Document Name:", iter)
print("CheckList Rule - 22: Probable Acronyms in Document not defined in Acronym and Definition Table.")
print("Document Review Start Time:", start,"HH:MM:SS")
print("--------------------------------------------------------------------------------------------------------")
print("\n")
app=[]
if iter.endswith('.doc') or iter.endswith('.docx'):
 word1 = win32.Dispatch("Word.Application")
 word1.Visible = True
 p = os.path.abspath(iter)
 word1.Documents.Open(p)
 sheet_1 = word1.ActiveDocument
 for para in sheet_1.Paragraphs:
  a=para.Range.Text.encode('ascii','ignore').decode()
  pattern = r'(?:[A-Z]\.)+'
  t = re.findall('([A-Z]+)',a)
  for i in t:
   if len(i)>=2:     #discarding single capital letters
    app.append(i)
 def get_table_count():
  return sheet_1.Tables.Count

 def count_table_rows(table):
  return table.Rows.Count

 def count_table_columns(table):
  return table.Columns.Count

 def get_headers():
  headers = sheet_1.Sections(1).Headers(1)
  shape_count = headers.Shapes.Count
  for shape_num in range(1, shape_count + 1):
      t_range = headers.Shapes(shape_num).TextFrame.TextRange
      text = t_range.Text
      page_num = t_range.Information(3)  # 3 == wdActiveEndPageNumber
      yield text, page_num

 def get_table_text(table):
  col_count = count_table_columns(table)
  row_count = count_table_rows(table)

  for row in range(1, row_count + 1):
      row_data = []
      for col in range(1, col_count + 1):
          try:
              row_data.append(table.Cell(Row=row,Column=col).Range.Text.strip(chr(7) + chr(13)))

          except pythoncom.com_error as error:
              row_data.append("")

      yield row_data

 def get_all_table_text():
  for table in get_tables():
      table_data = []
      for row_data in get_table_text(table):
          #for col_data in .get_table_text(table):
              #table_data1.append(col_data)
              table_data.append(row_data)
      yield table_data
      #yield table_data1

 def get_tables():
  for table in sheet_1.Tables:
      yield table

 def __del__():
  word1.Quit()
 try:
  res=[]
  res1=[]
  pp={}
  jj=[]
  jjj=[]
  jjjj=[]

         #path = str(input())
         #count=0
         #open_doc = os.path.abspath(path)
  for table_num, table_text in enumerate(get_all_table_text()):
      #print("\n-------------- Table %s ----------------" % (table_num + 1))
      for row_data in table_text:
          b=", ".join(row_data)
          b=str(b).encode("utf-8")
          #print(b)
          k=b"Acronyms"
          l=b"Definition"
          if k in b:
              #print(table_text)
              k=table_text[0]
              #print(k)
              m=k.index('Acronyms')
              #print(m)
              for i in table_text:
               res.append(i[m])
  print("----------------------------------------------------------------------------------------------------------------------")
  print("Acronyms Defined:\n",res[1:])
  print("----------------------------------------------------------------------------------------------------------------------")
  res1=res[1:]
  print("Probable Acronyms not defined:\n",set(app)-set(res1))
  pp=set(app)-set(res1)
  #if len(pp)==0:
  # print("\nStatus:Pass")
  #else:
  # print("\nStatus:Fail")
 except:
  pass

end=datetime.now()
print("\nDocument Review End Time:", end)
print("\nTime taken For Document Review:", end-start,"HH:MM:SS")
sheet_1.Close()
word1.Quit()


from django.db import models
from django.conf import settings

models_yaml = settings.MODELS_YAML

models_dict = {}

types = {
    'date': models.DateField,
    'char': models.TextField,
    'int': models.IntegerField,
}

def model_name(title):
    def function(self):
        return title
    return function

for m in models_yaml:
    class_fields = {}
    class_fields['__module__'] = __name__
    class_fields['__unicode__'] = model_name(models_yaml[m]['title'])
    for f in models_yaml[m]['fields']:
        class_fields[f['id']] = types[f['type']](f['title'])

    models_dict[m] = type(m, (models.Model,), class_fields)


import operator
import os
import sys
import math
from flask import Flask
from flask_jsonrpc import JSONRPC

sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))

import mongodb_client

PREFERENCE_MODEL_TABLE_NAME = "user_preference_model"

SERVER_HOST = 'localhost'
SERVER_PORT = 5050

# Flask application
app = Flask(__name__)

# Flask-JSONRPC
jsonrpc = JSONRPC(app, '/api', enable_web_browsable_api=True)

@jsonrpc.method('getPreferenceForUser')
def getPreferenceForUser(user_id:str)->list:
    db = mongodb_client.get_db()
    model = db[PREFERENCE_MODEL_TABLE_NAME].find_one({'userId':user_id})
    print(model)
    if model is None:
        return []
    sorted_tuples = sorted(model['preference'].items(), key=operator.itemgetter(1), reverse=True)
    sorted_list = [x[0] for x in sorted_tuples]
    sorted_value_list = [x[1] for x in sorted_tuples]

    # If the first preference is close to the last one, return empty
    if math.isclose(float(sorted_value_list[0]), float(sorted_value_list[-1]), rel_tol=1e-5):
        return []
    print(sorted_list)
    return sorted_list

if __name__ == '__main__':
    app.run(host=SERVER_HOST, port=SERVER_PORT, debug=True)


# -*- coding: utf-8 -*-
import time
from odoo import api, models, _
from odoo.exceptions import UserError
from datetime import datetime

class ReportCostProfit(models.AbstractModel):

    _name = 'report.staffcommission_costprofit.report_cost_profit'

    def get_cost_profit_data(self, start_date, end_date, doctor, treatment_id):
        dom = [('date_invoice', '>=', start_date),
              ('date_invoice', '<=', end_date),
              ('is_patient', '=', True),
              ('type', '=', 'out_invoice'),
              ('state', 'in', ['open', 'paid'])]
        if doctor:
            dom.append(('dentist', '=', doctor[0]))
        history_ids = self.env['account.invoice'].search(dom)
        prod_dict = {}
        for invoice in history_ids:
            income = 0.0
            overhead_amt = 0.0
            material_cost = 0.0
            if invoice:
                for line in invoice.invoice_line_ids:
                    count = 0
                    if line.product_id.is_treatment:
                        if (treatment_id and line.product_id.id == treatment_id[0]) or not treatment_id:
                            income = line.price_subtotal
                            overhead_amt = line.product_id.lst_price * (line.product_id.overhead_cost/100)
                            material_cost = 0.0
                            for consumable in line.product_id.consumable_ids:
                                material_cost += consumable.consu_product_id.lst_price * consumable.quantity
                            if line.product_id.id in prod_dict:
                                count = prod_dict[line.product_id.id][1] + line.quantity
                                prod_dict[line.product_id.id][1] = count
                                overhead_final = overhead_amt * line.quantity
                                material_cost_final = material_cost * line.quantity
                                prod_dict[line.product_id.id][2] += income
                                prod_dict[line.product_id.id][3] += overhead_final
                                prod_dict[line.product_id.id][4] += material_cost_final
                                prod_dict[line.product_id.id][5] += income - material_cost_final - overhead_final
                            else:
                                overhead_final = overhead_amt * line.quantity
                                material_cost_final = material_cost * line.quantity
                                prod_dict[line.product_id.id] = [line.product_id.name, line.quantity,
                                                                 income, overhead_final, material_cost_final,
                                                                 income-material_cost_final -overhead_final]
        return [prod_dict]

    @api.multi
    def get_report_values(self, docids, data=None):
        if not data.get('form') or not self.env.context.get('active_model') or not self.env.context.get('active_id'):
            raise UserError(_("Form content is missing, this report cannot be printed."))
        model = self.env.context.get('active_model')
        docs = self.env[model].browse(self.env.context.get('active_id'))
        start_date = data['form']['date_start']
        end_date = data['form']['date_end']
        treatment_id = data['treatment_id']
        doctor = data['doctor']
        final_records = self.get_cost_profit_data(start_date, end_date, doctor, treatment_id)
        period_start = datetime.strptime(start_date, '%Y-%m-%d')
        period_stop = datetime.strptime(end_date, '%Y-%m-%d')
        return {
            'period_start': period_start,
            'period_stop': period_stop,
            'doc_ids': self.ids,
            'doc_model': 'cost.profit.wizard',
            'data': data['form'],
            'docs': docs,
            'time': time,
            'get_cost_profit_data': final_records,
            'treatment_id': treatment_id,
            'doctor': doctor,
        }

    def formatLang(self, value, digits=None, date=False, date_time=False, grouping=True, monetary=False, dp=False,
                   currency_obj=False, lang=False):
        if lang:
            self.env.context['lang'] = lang
        return super(ReportCostProfit, self).formatLang(value, digits=digits, date=date, date_time=date_time,
                                                               grouping=grouping, monetary=monetary, dp=dp,
                                                               currency_obj=currency_obj)


try:
    import ameba
    import time
    import uio as io
    import array
    import uctypes
    from utils import fast_copy
except:
    raise ImportError("ameba module import failed")

PERI_ON_BASE = const(0x40000000)
REG_CPU_PERIPHERAL_CTRL = const(0x02C0)
SPI_FLASH_BASE = const(0x98000000)
SPI_FLASH_CTRL_BASE = const(0x40006000)

REG_SPIC_CTRLR0 = const(0x0000)
REG_SPIC_CTRLR1 = const(0x0004)
REG_SPIC_SSIENR = const(0x0008)
REG_SPIC_SR = const(0x0028)
REG_SPIC_DR0 = const(0x0060)
REG_SPIC_ADDR_LENGTH = const(0x0118)

class FlashBdev:
    START_ADDR = 0x100000
    SEC_SIZE = 0x1000 # most flash sector is 4K
    START_SEC = START_ADDR // SEC_SIZE
    NUM_BLK = 252
    def __init__(self, blocks=NUM_BLK):
        self.blocks = blocks
        self.cache_buffer = bytearray(self.SEC_SIZE)

    def cmd(self, cmd, data=None, rxbuf=None):
        if len(cmd) != 1:
            raise ValueError('cmd length should be 1 byte')

        tmod = 1 if rxbuf is None else 3

        # enable CPU's spiflash pin
        ameba.mem32[PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL] |= (1<<0)
        ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_SSIENR] &= ~(1<<0)
        if tmod == 3:
            ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_CTRLR1] = len(rxbuf)
        if data is not None:
            ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_ADDR_LENGTH] = 0 if data is None else len(data)

        ctrlr0 = ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_CTRLR0]
        ctrlr0 &= 0xFFC0FCFF
        ctrlr0 |= ((tmod << 8) | (3 << 20))
        ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_CTRLR0] = ctrlr0

        ameba.mem8[SPI_FLASH_CTRL_BASE + REG_SPIC_DR0] = int.from_bytes(cmd, 1)

        if data is not None:
            for i in data:
                ameba.mem8[SPI_FLASH_CTRL_BASE + REG_SPIC_DR0] = i

        # activate this transfer
        ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_SSIENR] |= (1<<0)

        counter = 1000000
        while counter:
            reg_sr = ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_SR]
            busy = reg_sr & (1<<0)
            if busy == 0:
                break
            counter -= 1

        ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_SSIENR] &= ~(1<<0)

        if rxbuf is not None:
            for i in range(len(rxbuf)):
                rxbuf[i] = ameba.mem8[SPI_FLASH_CTRL_BASE + REG_SPIC_DR0]

        ameba.mem32[SPI_FLASH_CTRL_BASE + REG_SPIC_SSIENR] &= ~(1<<0)
        # disable CPU's spiflash pin
        ameba.mem32[PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL] &= ~(1<<0)

    def WREN(self):
        self.cmd(b'\x06')

    def WRDI(self):
        self.cmd(b'\x04')

    def WRSR(self, status):
        self.cmd(b'\x01', data=status)

    @property
    def RDSR(self):
        status = bytearray(1)
        self.cmd(b'\x05', rxbuf=status)
        return int.from_bytes(status, 1)

    @property
    def WIP(self):
        status = self.RDSR
        return status & (1<<0)

    @property
    def RDID(self):
        buffer = bytearray(3)
        self.cmd(b'\x9f', rxbuf=buffer)
        id = int.from_bytes(buffer, 3)
        manufactor_id = (id & 0xFF0000) >> 16
        device_id = id & 0xFFFF
        return (manufactor_id, device_id)

    def erasesector(self, addr):
        bytes_list = list(addr.to_bytes(3, 'little'))
        bytes_list.reverse()
        self.cmd(b'\x20', data = bytearray(bytes_list))

    def eraseblock(self, addr):
        bytes_list = list(addr.to_bytes(3, 'little'))
        bytes_list.reverse()
        self.cmd(b'\xd8', data = bytearray(bytes_list))

    def erasechip(self):
        self.cmd(b'\x60')

    def readblocks(self, n, buf):
        flash_cpu_ctrl_address = PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL
        ameba.mem32[flash_cpu_ctrl_address] |= (1<<0)
        dst_addr = uctypes.addressof(buf)
        src_addr = SPI_FLASH_BASE + (n + self.START_SEC) * self.SEC_SIZE
        fast_copy(dst_addr, src_addr, self.SEC_SIZE)
        ameba.mem32[flash_cpu_ctrl_address] &= ~(1<<0)

    def writeblocks(self, n, buf):
        address_base = SPI_FLASH_BASE + (n + self.START_SEC) * self.SEC_SIZE
        spi_status_address = SPI_FLASH_CTRL_BASE + REG_SPIC_SR
        flash_cpu_ctrl_address = PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL

        ameba.mem32[PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL] |= (1<<0)
        self.WREN()
        self.erasesector((n + self.START_SEC) * self.SEC_SIZE)
        while self.WIP != 0:
            pass

        arr = array.array("L", buf)

        for index, value in enumerate(arr):
            ameba.mem32[flash_cpu_ctrl_address] |= 0x01
            ameba.mem32[address_base + 4*index] = value

        ameba.mem32[PERI_ON_BASE + REG_CPU_PERIPHERAL_CTRL] &= ~(1<<0)

    def ioctl(self, op, arg):
        if op == 4: # BP_IOCTL_SEC_COUNT
            return self.blocks
        if op == 5: # BP_IOCTL_SEC_SIZE
            return self.SEC_SIZE


import logging
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pickle as pkl
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import torch
import torch.utils.data as utils

import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/..")

from common.argparser import autoencoder_argparse
from common.utility import log_shapes, log_time, torch_device,\
    time_stp, load_processed_data, logger, sep
from common.torchsummary import summary

from models.autoencoder import AutoEncoderBasic

def main(
        model,
        time_stamp,
        device,
        ally_classes,
        advr_1_classes,
        advr_2_classes,
        encoding_dim,
        test_size,
        batch_size,
        n_epochs,
        shuffle,
        lr,
        expt,
        ):

    device = torch_device(device=device)

    # refer to PrivacyGAN_Titanic for data preparation
    X, y_ally, y_advr_1, y_advr_2 = load_processed_data(
        expt, 'processed_data_X_y_ally_y_advr_y_advr_2.pkl')
    log_shapes(
        [X, y_ally, y_advr_1, y_advr_2],
        locals(),
        'Dataset loaded'
    )

    X_train, X_valid = train_test_split(
            X,
            test_size=test_size,
            stratify=pd.DataFrame(np.concatenate(
                (
                    y_ally.reshape(-1, ally_classes),
                    y_advr_1.reshape(-1, advr_1_classes),
                    y_advr_2.reshape(-1, advr_2_classes),
                ), axis=1)
            )
        )

    log_shapes(
        [
            X_train, X_valid,
        ],
        locals(),
        'Data size after train test split'
    )

    scaler = StandardScaler()
    X_train_normalized = scaler.fit_transform(X_train)
    X_valid_normalized = scaler.transform(X_valid)

    log_shapes([X_train_normalized, X_valid_normalized], locals())

    dataset_train = utils.TensorDataset(torch.Tensor(X_train_normalized))
    dataloader_train = torch.utils.data.DataLoader(
        dataset_train, batch_size=batch_size, shuffle=shuffle, num_workers=2)

    dataset_valid = utils.TensorDataset(torch.Tensor(X_valid_normalized))
    dataloader_valid = torch.utils.data.DataLoader(
        dataset_valid, batch_size=batch_size, shuffle=False, num_workers=2)

    auto_encoder = AutoEncoderBasic(
        input_size=X_train_normalized.shape[1],
        encoding_dim=encoding_dim
    ).to(device)

    criterion = torch.nn.MSELoss()
    adam_optim = torch.optim.Adam
    optimizer = adam_optim(auto_encoder.parameters(), lr=lr)

    summary(auto_encoder, input_size=(1, X_train_normalized.shape[1]))

    h_epoch = []
    h_valid = []
    h_train = []

    auto_encoder.train()

    sep()
    logging.info("epoch \t Aencoder_train \t Aencoder_valid")

    for epoch in range(n_epochs):

        nsamples = 0
        iloss = 0
        for data in dataloader_train:
            optimizer.zero_grad()

            X_torch = data[0].to(device)
            X_torch_hat = auto_encoder(X_torch)
            loss = criterion(X_torch_hat, X_torch)
            loss.backward()
            optimizer.step()

            nsamples += 1
            iloss += loss.item()

        if epoch % int(n_epochs/10) != 0:
            continue

        h_epoch.append(epoch)
        h_train.append(iloss/nsamples)

        nsamples = 0
        iloss = 0
        for data in dataloader_valid:
            X_torch = data[0].to(device)
            X_torch_hat = auto_encoder(X_torch)
            loss = criterion(X_torch_hat, X_torch)

            nsamples += 1
            iloss += loss.item()
        h_valid.append(iloss/nsamples)

        logging.info('{} \t {:.8f} \t {:.8f}'.format(
            h_epoch[-1],
            h_train[-1],
            h_valid[-1],
        ))

    config_summary = 'device_{}_dim_{}_batch_{}_epochs_{}_lr_{}_tr_{:.4f}_val_{:.4f}'\
        .format(
            device,
            encoding_dim,
            batch_size,
            n_epochs,
            lr,
            h_train[-1],
            h_valid[-1],
        )

    plt.plot(h_epoch, h_train, 'r--')
    plt.plot(h_epoch, h_valid, 'b--')
    plt.legend(['train_loss', 'valid_loss'])
    plt.title("autoencoder training {}".format(config_summary))

    plot_location = 'plots/{}/{}_training_{}_{}.png'.format(
        expt, model, time_stamp, config_summary)
    sep()
    logging.info('Saving: {}'.format(plot_location))
    plt.savefig(plot_location)
    checkpoint_location = \
        'checkpoints/{}/{}_training_history_{}_{}.pkl'.format(
            expt, model, time_stamp, config_summary)
    logging.info('Saving: {}'.format(checkpoint_location))
    pkl.dump((h_epoch, h_train, h_valid), open(checkpoint_location, 'wb'))

    model_ckpt = 'checkpoints/{}/{}_torch_model_{}_{}.pkl'.format(
            expt, model, time_stamp, config_summary)
    logging.info('Saving: {}'.format(model_ckpt))
    torch.save(auto_encoder, model_ckpt)

if __name__ == "__main__":
    expt = 'mimic'
    model = 'autoencoder_basic'
    marker = 'A'
    pr_time, fl_time = time_stp()

    logger(expt, model, fl_time, 'A')

    log_time('Start', pr_time)
    args = autoencoder_argparse()
    main(
        model=model,
        time_stamp=fl_time,
        device=args['device'],
        ally_classes=int(args['n_ally']),
        advr_1_classes=int(args['n_advr_1']),
        advr_2_classes=int(args['n_advr_2']),
        encoding_dim=int(args['dim']),
        test_size=float(args['test_size']),
        batch_size=int(args['batch_size']),
        n_epochs=int(args['n_epochs']),
        shuffle=int(args['shuffle']) == 1,
        lr=float(args['lr']),
        expt=args['expt'],
    )
    log_time('End', time_stp()[0])
    sep()


# 对 content.txt 中的每一条数据进行分词
# 分词过后用 ' ' 拼接起来，还是作为一条数据存放在 content-split-timestamp.txt 文件中
import time
import multiprocessing
import jieba
import codecs

if __name__ == '__main__':
  curTimeStr = str(int(time.time()))
  stopwords = []

  with open('./stop-words.txt', 'r') as f:
    stopwords = [word.strip() for word in f.readlines()]
    f.close()

  # 去除停用词
  def remove_stopwords(ls):
    return [word for word in ls if word not in stopwords]

  # 以写的方式打开原始的简体中文语料库
  source_f = codecs.open('./content.txt', 'r', encoding='utf8')
  # 将分完词的语料写入到 content-split-timestamp.txt 文件中
  target_f = codecs.open('./content-split-' + curTimeStr + '.txt', 'w', encoding='utf8')

  line = source_f.readline()

  # 设置 jieba 分词操作为并行
  # parallel 数量为 cpu_count
  jieba.enable_parallel(multiprocessing.cpu_count())
  # jieba 加载自定义词典
  # 目前自定义词典来源
  # https://github.com/qbox/www/tree/2020/front/2020/constants 中的 products && solutions 目录
  jieba.load_userdict('./user-dict.txt')

  while line:
    # 排除首尾空格 && 换行
    # 统一转换为小写
    line = line.strip().lower()
    line_seg = ' '.join(remove_stopwords([word for word in jieba.cut(line)]))
    target_f.writelines(line_seg + '\n')

    line = source_f.readline()

  # 关闭两个文件流
  source_f.close()
  target_f.close()


import pandas
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, precision_recall_curve
import numpy as np

data = pandas.read_csv('data_seta/scores.csv')

y_true = data['true'].to_numpy()
y_pred = data['pred'].to_numpy()

TP = 0
TN = 0
FP = 0
FN = 0

for i in range(0, len(y_true)):
    if y_true[i] == 1 and y_pred[i] == 1:
        TP += 1
    if y_true[i] == 0 and y_pred[i] == 1:
        FP += 1
    if y_true[i] == 1 and y_pred[i] == 0:
        FN += 1
    if y_true[i] ==0 and y_pred[i] == 0:
        TN += 1

accuracy = accuracy_score(y_true, y_pred)
precision = precision_score(y_true, y_pred)
recall = recall_score(y_true, y_pred)
f = f1_score(y_true, y_pred)

classifiers = pandas.read_csv('data_seta/classification.csv')

y1_true = classifiers['true'].to_numpy()
score_logreg = classifiers['score_logreg'].to_numpy()
score_svm = classifiers['score_svm'].to_numpy()
score_knn = classifiers['score_knn'].to_numpy()
score_tree = classifiers['score_tree'].to_numpy()

logreg = roc_auc_score(y1_true, score_logreg)
svm = roc_auc_score(y1_true, score_svm)
knn = roc_auc_score(y1_true, score_knn)
tree = roc_auc_score(y1_true, score_tree)

prc_logred = precision_recall_curve(y1_true, score_logreg)
prc_svm = precision_recall_curve(y1_true, score_svm)
prc_knn = precision_recall_curve(y1_true, score_knn)
prc_tree = precision_recall_curve(y1_true, score_tree)

def find_biggest_precision_when_recall(array):
    precision, recall, thresholds = array

    recall_indx = np.where(recall > 0.7)
    filtered_precision = precision[recall_indx]

    return round(max(filtered_precision), 2)

print(find_biggest_precision_when_recall(prc_logred))
print(find_biggest_precision_when_recall(prc_svm))
print(find_biggest_precision_when_recall(prc_knn))
print(find_biggest_precision_when_recall(prc_tree))


import os
import packaging.version

from testconfig import config

from unittest import skip

from tests.integration.core.constants import UPDATE_TEST_TIMEOUT
from tests.integration.installation_and_upgrade.test_installation_and_upgrade import TestInstallationAndUpgrade

class TestYumUpdate(TestInstallationAndUpgrade):
    def test_yum_update(self):
        """ Test for lustre kernel is set to boot after yum update"""
        self.assertGreaterEqual(len(config["lustre_servers"]), 4)

        # get a list of hosts
        response = self.chroma_manager.get("/api/host/")
        self.assertEqual(response.successful, True, response.text)
        hosts = response.json["objects"]
        self.assertEqual(len(hosts), len(self.config_servers))

        if packaging.version.parse(os.environ["UPGRADE_FROM_VER"]) > packaging.version.parse("4.1"):
            # Ensure that IML notices its storage servers needs upgraded
            for host in hosts:
                # wait for an upgrade available alert
                self.wait_for_assert(lambda: self.assertHasAlert(host["resource_uri"], of_type="UpdatesAvailableAlert"))
                alerts = self.get_list("/api/alert/", {"active": True, "alert_type": "UpdatesAvailableAlert"})

                # Should be the only alert
                # FIXME HYD-2101 have to filter these alerts to avoid spurious ones.  Once that
                # ticket is fixed, remove the filter so that we are checking that this really is
                # the only alert systemwide as it should be.
                alerts = [a for a in alerts if a["alert_item"] == host["resource_uri"]]
                self.assertEqual(len(alerts), 1)

                # Should be an 'updates needed' alert
                self.assertRegexpMatches(alerts[0]["message"], "Updates are ready.*")

                # The needs_update flag should be set on the host
                self.assertEqual(self.get_json_by_uri(host["resource_uri"])["needs_update"], True)

            self.assertEqual(len(hosts), len(self.config_servers))

        # Even though we have to stop a filesytem to do an upgrade (i.e. no
        # rolling upgrades, we stopped it before doing the upgrade to avoid
        # situations where the O/S upgrade results in an IML that can no
        # longer function with the upgraded O/S.

        # Run a yum upgrade while excluding python2 packages
        for server in self.config_servers:
            self.remote_operations.yum_upgrade_exclude_python2_iml(server)

        # With the list of hosts, start the upgrade as a single command
        command = self.chroma_manager.post(
            "/api/command/",
            body={
                "jobs": [{"class_name": "UpdateJob", "args": {"host_id": host["id"]}} for host in hosts],
                "message": "Test update of hosts",
            },
        ).json

        # doing updates can include a reboot of the storage server,
        # and perhaps RHN/proxy slowness, so give it some extra time
        # Also note that IML is internally updating nodes in the same
        # HA pair in serial, so the timeout needs to be 2x.
        self.wait_for_command(self.chroma_manager, command["id"], timeout=UPDATE_TEST_TIMEOUT)
        self.wait_for_assert(lambda: self.assertNoAlerts(host["resource_uri"], of_type="UpdatesAvailableAlert"))

        for server in self.config_servers:
            kernel = self.remote_operations.default_boot_kernel_path(server)
            self.assertGreaterEqual(kernel.find("_lustre"), 7)
            available_updates = self.remote_operations.yum_check_update(server)
            available_updates = filter(lambda x: "iml-" in x, available_updates)
            self.assertEqual(len(available_updates), 0)

        # Update corosync on the storage servers
        # N.B. This will also start the FS
        for server in self.config_servers:
            self.remote_command(server["fqdn"], "chroma-agent convert_targets")

        # Start the filesystem back up
        filesystem = self.get_filesystem_by_name(self.fs_name)
        self.start_filesystem(filesystem["id"])

    @skip("Repos can't really be retired until at least an n+1 release")
    def test_no_retired_repos(self):
        "Test that no retired repos exist after an upgrade"
        # TODO: this really needs to be more dynamic than using
        #       repos that would have been retired many re-
        #       leases ago
        retired_repos = ["robinhood"]
        current_repos = self.remote_operations.get_chroma_repos()
        for repo in retired_repos:
            self.assertFalse(repo in current_repos, "Unexpectedly found repo '%s' in %s" % (repo, current_repos))

    def test_obsolete_chroma_diagnostics(self):
        """Test that chroma-diagnostics has been obsoleted"""
        addresses = [server["fqdn"] for server in self.config_servers]
        addresses.append(config["chroma_managers"][0]["address"])

        for address in addresses:
            chroma_diagnostics_result = self.remote_command(address, "chroma-diagnostics")
            self.assertEqual(
                chroma_diagnostics_result.stdout.split("\n")[0],
                "chroma-diagnostics no longer exists. Please use 'iml-diagnostics' instead.",
            )

            yum_installed_result = self.remote_command(address, "rpm -q chroma-diagnostics", expected_return_code=1)
            self.assertNotEqual(yum_installed_result.exit_status, 0)

    # something we can run to clear the storage targets since this
    # test class doesn't use setUp()
    def test_clean_linux_devices(self):
        self.cleanup_linux_devices([s for s in self.TEST_SERVERS if "worker" not in s.get("profile", "")])

    def test_stop_before_update(self):
        # Stop the filesystem. Currently the GUI forces you to stop the filesystem before
        # the buttons to install updates is available as we don't do a kind "rolling upgrade".
        filesystem = self.get_filesystem_by_name(self.fs_name)
        if filesystem["state"] != "stopped":
            self.stop_filesystem(filesystem["id"])


import os
import time
import shutil
import pickle

from common.entity.entities import Command, Node, Type
from common.network.utils import get_ip_address
from common.network.utils import changeIp
from sftp import download_from_ftp

class BBB():

    def __init__(self, path, rc_local_dest_path, sfp_server_addr, sftp_port: int = 22,
                 ftp_destination_folder: str = None):

        #  Parameters that define absolute locations inside the Beaglebone
        self.ftpDestinationFolder = ftp_destination_folder
        self.rcLocalDestPath = rc_local_dest_path
        self.configPath = path
        self.sfp_server_addr = sfp_server_addr
        self.sftp_port = sftp_port

        # The interface used (ethernet port on the Beaglebone).
        self.interfaceName = 'eth0'

        # My configs !
        self.node = Node()
        self.node.type = Type()
        self.node.ipAddress = get_ip_address(self.interfaceName)

        # Load the data from the cfg file.
        self.readParameters()

    def getInfo(self):
        info = "{}|{}|{}|{}|{}" \
            .format(Command.PING, self.node.name, self.node.type.name, self.node.ipAddress, self.node.type.sha)
        return info

    def reboot(self):
        os.system('reboot')

    def update_project(self):
        """
            Update the project, getting the current version available on the FTP server.
            The responsibility to keep it up to date is all on the server side.
            Always removing the old ones.
        :return: True or False
        """
        if type is None:
            return False
        try:
            repo_name = self.node.type.repoUrl.strip().split('/')[-1].split('.')[0]

            repo_dir = self.ftpDestinationFolder + repo_name + "/"
            if os.path.exists(repo_dir) and os.path.isdir(repo_dir):
                shutil.rmtree(repo_dir)
                time.sleep(1)

            if repo_dir.endswith('/') and self.node.rcLocalPath.startswith('/'):
                self.node.rcLocalPath = self.node.rcLocalPath[1:]

            download_from_ftp(sftp_server_addr=self.sfp_server_addr, sftp_port=self.sftp_port, path=repo_name,
                              destination=repo_dir)

            print("Downloaded Node repository from FTP server {} at {}".format(self.node.type.repoUrl, repo_name))

            if not os.path.isfile(repo_dir + self.node.rcLocalPath):
                shutil.rmtree(repo_dir)
                raise Exception("rc.local not found on path.")

            shutil.copy2((repo_dir + self.node.rcLocalPath), self.rcLocalDestPath)
            print("Copied file {} to {}".format(repo_dir + self.node.rcLocalPath, self.rcLocalDestPath))
            return True
        except Exception as e:
            print("{}".format(e))
            return False

    def update(self, desiredNode: Node = None):
        """
        @todo : NODE! Update Code !!!!
        Update the bbb with new data and refresh the project.
        :return:
        """
        if not desiredNode or not desiredNode.type:
            print('Node/Type is None !')
            return
        if not desiredNode.ipAddress:
            print('IP Address is None !')
            return
        try:
            self.node = desiredNode

            res, msg = changeIp(interface_name=self.interfaceName, desired_ip=desiredNode.ipAddress)
            if res:
                self.node.ipAddress = get_ip_address(self.interfaceName)
            print(msg)

            with open("/etc/hostname", "w") as hostnameFile:
                hostnameFile.write(desiredNode.name.replace(":", "-"))
                hostnameFile.close()

            self.update_project()

            self.writeNodeConfig(pickle.dumps(self.node))
        except Exception as e:
            print('Update Error ! {}'.format(e))

    def readParameters(self):
        self.readNodeConfig()
        try:
            name = os.popen("hostname", "r").readline()[:-1]
            indexes = [i for i, letter in enumerate(name) if letter == "-"]
            name = list(name)
            if len(indexes) > 2:
                name[indexes[1]] = ":"

            self.node.name = "".join(name)
        except FileNotFoundError:
            self.node.name = "error-hostname-not-found"

    def readNodeConfig(self):
        try:
            with open(self.configPath, 'rb') as file:
                self.node = pickle.load(file)
        except Exception as e:
            print('Read node config exception {}'.format(e))

    def writeNodeConfig(self, data):
        with open(self.configPath, 'wb+') as file:
            file.write(data)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
    File name: main.py
    Location: Bioinformatics armory
    Project: Rosalind.info
    Author: Alexander Popp
    Date created: 08/26/2018
    Date last modified: 8/26/2018
    Python version: 3.6.5
    Description:
        DBPR: Given the uniprot ID of a protein, return a list of biological
        processes in which the protein is involved (biological processes are
        found in a subsection of the protein's 'Gene Ontology' (GO) section).
"""

from Bio import ExPASy
from Bio import SwissProt

# Read input file
def read_file(file_name):
    with open(file_name, 'r') as f:
        return f.readline()

# Write output file
def write_file(file_name, data):
    with open(file_name, 'w') as f:
        for item in data:
            f.write(str(item) + "\n")

# Returns list of biological processes
def get_processes(prot_id):
    handle = ExPASy.get_sprot_raw(prot_id)
    record = SwissProt.read(handle)
    repos = record.cross_references
    result = []

    for i in range(len(repos)):
        if 'GO' in repos[i]:
            if 'P:' in repos[i][2]:
                result.append(repos[i][2][2::])
    result = map(str, result)
    return list(result)

def main():
    result = get_processes(read_file('rosalind_dbpr.txt'))
    write_file('output_dbpr.txt', result)

if __name__ == "__main__":
    main()


from __future__ import print_function

import sys
import cdp.cdp_parameter

class ACMEParameter(cdp.cdp_parameter.CDPParameter):
    def __init__(self):
        self.case_id = ''
        self.reference_data_path = ''
        self.test_data_path = ''
        self.viewer_descr = {}

        self.sets = []
        self.dataset = ''
        self.run_type = 'model_vs_obs'
        self.variables = []
        self.seasons = ['ANN', 'DJF', 'MAM', 'JJA', 'SON']
        self.regions = ['global']
        self.regrid_tool = 'esmf'
        self.regrid_method = 'conservative'
        self.plevs = []

        # Plotting related
        self.main_title = ''
        # self.backend = 'vcs'  # No default backend for now, user needs to specify which one
        self.save_netcdf = False
        self.output_format = ['png']
        self.output_format_subplot = []
        self.canvas_size_w = 1212
        self.canvas_size_h = 1628
        self.figsize = [8.5, 11.0]
        self.dpi = 150
        self.arrows = True
        self.logo = False

        self.contour_levels = []  # used both in test and reference
        self.test_name = ''
        self.short_test_name = ''
        self.test_title = ''
        # self.test_colormap = 'viridis'
        self.test_colormap = 'cet_rainbow.rgb'
        self.test_units = ''

        self.reference_name = ''
        self.reference_title = ''
        # self.reference_colormap = 'viridis'
        self.reference_colormap = 'cet_rainbow.rgb'
        self.reference_units = ''

        self.diff_name = ''
        self.diff_title = 'Model - Observation'
        # self.diff_colormap = 'cet_diverging_bwr_55_98_c37'
        self.diff_colormap = 'diverging_bwr.rgb'
        self.diff_levels = []
        self.diff_units = ''

        self.multiprocessing = False
        self.distributed = False
        self.num_workers = 4

        self.no_viewer = False
        self.debug = False

        self.granulate = ['variables', 'seasons', 'regions', 'plevs']

    def check_values(self):
        if not hasattr(
                self, 'reference_data_path') or self.reference_data_path == '':
            print('You need to specify reference_data_path in the parameters file or in the command line using --reference_data_path')
            sys.exit()
        if not hasattr(self, 'test_data_path') or self.test_data_path == '':
            print('You need to specify test_data_path in the parameters file or in the command line using --test_data_path')
            sys.exit()
        if hasattr(self, 'multiprocessing') and hasattr(
                self, 'distributed') and self.multiprocessing and self.distributed:
            print("Why are you trying to run the diags multiprocessed and distributedly? You can't do this, only choose one or none.")
            sys.exit()
        if not hasattr(self, 'backend'):
            print("You need to define the 'backend' parameter to 'vcs' or 'mpl'/'matplotlib'/'cartopy'.")
            sys.exit()


import plotly
import plotly.graph_objs as go
import json

def create_plot(meters, prediction):
    reading_0 = prediction["reading"].loc[prediction["meter"] == 0]
    reading_1 = prediction["reading"].loc[prediction["meter"] == 1]
    reading_2 = prediction["reading"].loc[prediction["meter"] == 2]
    reading_3 = prediction["reading"].loc[prediction["meter"] == 3]
    timestamp = prediction["timestamp"]

    data = []

    if meters[0]:
        data.append(go.Scatter(x=timestamp, y=reading_0,
                               mode='lines+markers',
                               name='Electricity',
                               line=dict(color='darkolivegreen'),
                               showlegend=False))
    if meters[1]:
        data.append(go.Scatter(x=timestamp, y=reading_1,
                               mode='lines+markers',
                               name='Chilled Water',
                               line=dict(color='aqua'),
                               showlegend=False))
    if meters[2]:
        data.append(go.Scatter(x=timestamp, y=reading_2,
                               mode='lines+markers',
                               name='Steam',
                               line=dict(color='aquamarine'),
                               showlegend=False))
    if meters[3]:
        data.append(go.Scatter(x=timestamp, y=reading_3,
                               mode='lines+markers',
                               name='Hot Water',
                               line=dict(color='darkturquoise'),
                               showlegend=False))
    layout = {
        "height": 200,
        "margin": go.layout.Margin(
            t=20,
            b=30
        ),
        "paper_bgcolor": "transparent",
        "yaxis": {
            "linecolor": "#001f07",
            "mirror": True,
            "title": "Energy Consumption"
        },
        "xaxis": {
            "linecolor": "#001f07",
            "mirror": True,
            "tickformat": "%H:00 - %b %d",
            "tickmode": "auto",
            "dtick": "H3"
        }
    }
    return json.dumps({"data": data, "layout": layout}, cls=plotly.utils.PlotlyJSONEncoder)


from rest_framework import permissions

class UpdateOwnProfile(permissions.BasePermission):
    """allow user to edit own profile"""

    def has_object_permission(self, request, view, obj):
        """check user is trying to edit profiel"""
        if request.method in permissions.SAFE_METHODS:
            return True

        return obj.id == request.user.id

class UpdateOwnStatus(permissions.BasePermission):
    """allow usr to upadate their own status"""

    def has_object_permission(self, request, view, obj):
        """used to check user is trying to upadte their own stsaus"""
        if request.method in permissions.SAFE_METHODS:
            return True
        return obj.user_profile.id == request.user.id