"""Module with Scenario entity."""

from datetime import datetime
from collections import namedtuple

from sqlalchemy import and_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Query
from sqlalchemy.sql.expression import func

from autostorage.database.model.scenario.scenario import Scenario as ScenarioRecord
from autostorage.database.model.scenario.view_state import (
    ScenarioViewState as ScenarioViewStateRecord,
    )
from autostorage.database.model.scenario.structure_state import (
    ScenarioStructureState as ScenarioStructureStateRecord,
    ScenarioStructureLock as ScenarioStructureLockRecord,
    )

from autostorage.core._base import _BaseClient
from autostorage.core.param_spec import Pagination

class ScenarioStructureLockedError(Exception):
    """Exception to prevent simultanious changes of the scenario."""

class Scenario(_BaseClient):
    """Class to work with autostorage Scenario."""

    def __init__(self, base, scenario_id):
        super(Scenario, self).__init__(base)
        self.__id = scenario_id

    @property
    def scenario_id(self):
        """ID of the scenario."""
        return self.__id

    def get_view_states(self, pagination=None):
        """Get view states of the scenario.

        View states are ordered from newest to oldest.

        :param pagination: instance of :class:`Pagination <autostorage.core.param_spec.Pagination>`.
        :returns: list of instances :class:`ScenarioViewState`.
        """
        with self.base.get_session() as session:
            scenario = session.query(ScenarioRecord).filter_by(scenario_id=self.__id).one()
            states_query = scenario.view_states
            if pagination:
                offset = pagination.page_index * pagination.items_per_page
                states_query = states_query.offset(offset).limit(pagination.items_per_page)

            states = []
            for state_record in states_query:
                state = ScenarioViewState(
                    scenario=self,
                    name=state_record.name,
                    description=state_record.description,
                    date=state_record.changed
                    )
                states.append(state)

        return states

    def rename(self, name):
        """Rename the scenario.

        :param name: new name for the scenario.
        :raises: AssertionError if new name is the same as current one.
        """
        current_state = self.get_view_states(pagination=Pagination(0, 1))[0]
        assert current_state.name != name, "New name is the same as current one"
        state_record = ScenarioViewStateRecord(
            scenario_id=self.__id,
            name=name,
            description=current_state.description,
            changed=datetime.utcnow()
            )

        with self.base.get_session() as session:
            session.add(state_record)
            session.commit()

    def change_description(self, description):
        """Change description of the scenario.

        :param description: new name for the scenario.
        :raises: AssertionError if new description is the same as current one.
        """
        current_state = self.get_view_states(pagination=Pagination(0, 1))[0]
        assert current_state.description != description, (
            "New description is the same as current one"
            )
        state_record = ScenarioViewStateRecord(
            scenario_id=self.__id,
            name=current_state.name,
            description=description,
            changed=datetime.utcnow()
            )

        with self.base.get_session() as session:
            session.add(state_record)
            session.commit()

    def get_structure(self, structure_filter=None):
        """Get structure of scenario by filter.

        :param structure_filter: instance of :class:`ScenarioStructureFilter
            <autostorage.core.scenario.param_spec.ScenarioStructureFilter>`.
        :returns: dictionary with tree_path as keys and node_ids as values.
        """
        subquery = Query([
            ScenarioStructureStateRecord.scenario_id,
            ScenarioStructureStateRecord.tree_path,
            func.max(ScenarioStructureStateRecord.changed).label('newest_change_date')
            ]).filter_by(
                scenario_id=self.__id
            )

        if structure_filter:
            if structure_filter.date:
                subquery = subquery.filter(
                    ScenarioStructureStateRecord.changed <= structure_filter.date)

            if structure_filter.tree_path:
                subquery = subquery.filter(ScenarioStructureStateRecord.tree_path.like(
                    "{0}-%".format(structure_filter.tree_path)
                    ))

        subquery = subquery.group_by(
            ScenarioStructureStateRecord.scenario_id,
            ScenarioStructureStateRecord.tree_path
            ).subquery()

        states_query = Query([
            ScenarioStructureStateRecord
            ]).join(
                subquery,
                and_(
                    ScenarioStructureStateRecord.scenario_id == subquery.columns.scenario_id,
                    ScenarioStructureStateRecord.tree_path == subquery.columns.tree_path,
                    ScenarioStructureStateRecord.changed == subquery.columns.newest_change_date
                )
            ).filter(
                ScenarioStructureStateRecord.enabled == True  # pylint: disable=singleton-comparison
            )

        with self.base.get_session() as session:
            bound_query = states_query.with_session(session)
            return {record.tree_path: record.node_id for record in bound_query}

    def _lock_structure(self):
        """Lock the structure to prevent changes.

        :raises: :class:`ScenarioStructureLockedError` exception if scenario is already locked.
        """
        scenario_id = self.__id
        lock_record = ScenarioStructureLockRecord(scenario_id=self.__id)
        try:
            with self.base.get_session() as session:
                session.add(lock_record)
                session.commit()
        except IntegrityError:
            raise ScenarioStructureLockedError(
                "Structure of scenario with id '{0}' has been already locked".format(scenario_id)
                )

    def _unlock_structure(self):
        """Unlock the structure so changes could be made."""
        with self.base.get_session() as session:
            session.query(ScenarioStructureLockRecord).filter_by(scenario_id=self.__id).delete()
            session.commit()

    def attach_structure(self, attachment_nodes):
        """Attach nodes structured as tree to the scenario structure.

        :param attachment_nodes: dictionary with tree path as keys and list of instances of
            :class:`_AttachmentNode <autostorage.core.scenario.param_spec._AttachmentNode>`
            as values.
        """
        scenario_id = self.__id
        changed_date = datetime.utcnow()

        def _create_node_records(child_path_pattern, tree_paths, attachment_nodes):
            """Helpful method to create records for new nodes.

            :param child_path_pattern: pattern to create tree_path for node. Must contain {0}.
            :param tree_paths: set of existent tree_paths.
            :attachment_nodes: list of instances of
                :class:`_AttachmentNode <autostorage.core.scenario.param_spec._AttachmentNode>`

            :returns: list of :class:`ScenarioStructureStateRecord
                <autostorage.database.model.scenario.structure_state.ScenarioStructureStateRecord>`.
            """
            node_records = []
            index = -1
            for attachment_node in attachment_nodes:
                index += 1
                tree_path = child_path_pattern.format(index)
                while tree_path in tree_paths:
                    index += 1
                    tree_path = child_path_pattern.format(index)

                node_record = ScenarioStructureStateRecord(
                    node_id=attachment_node.get_node_id(),
                    scenario_id=scenario_id,
                    tree_path=tree_path,
                    changed=changed_date,
                    )
                node_records.append(node_record)

                children_node_records = _create_node_records(
                    child_path_pattern=tree_path + '-{0}',
                    tree_paths=tree_paths,
                    attachment_nodes=attachment_node.get_children()
                    )
                node_records.extend(children_node_records)

            return node_records

        self._lock_structure()
        try:
            records_to_create = []

            current_tree_paths = set(self.get_structure().keys())
            for parent_tree_path, attachment_trees in attachment_nodes.items():
                if parent_tree_path is None:
                    path_pattern = "{0}"
                elif parent_tree_path not in current_tree_paths:
                    continue
                else:
                    path_pattern = parent_tree_path + "-{0}"

                records_to_create.extend(_create_node_records(
                    path_pattern, current_tree_paths, attachment_trees
                    ))

            with self.base.get_session() as session:
                session.add_all(records_to_create)
                session.commit()
        finally:
            self._unlock_structure()

    def detach_structure(self, tree_paths):
        """Detach nodes from the scenario structure.

        :param tree_paths: list of tree paths to detach.
        """
        scenario_id = self.__id
        changed_date = datetime.utcnow()

        self._lock_structure()
        try:
            current_structure = self.get_structure()
            records_to_create = []
            for tree_path in tree_paths:
                if tree_path not in current_structure:
                    continue

                state_record = ScenarioStructureStateRecord(
                    node_id=current_structure[tree_path],
                    scenario_id=scenario_id,
                    tree_path=tree_path,
                    changed=changed_date,
                    enabled=False
                    )

                records_to_create.append(state_record)
                current_structure.pop(tree_path)

                child_path_prefix = tree_path + '-'
                current_paths = tuple(current_structure.keys())
                for path in current_paths:
                    if path.startswith(child_path_prefix):
                        state_record = ScenarioStructureStateRecord(
                            node_id=current_structure[path],
                            scenario_id=scenario_id,
                            tree_path=path,
                            changed=changed_date,
                            enabled=False
                            )
                        records_to_create.append(state_record)
                        current_structure.pop(path)

            with self.base.get_session() as session:
                session.add_all(records_to_create)
                session.commit()
        finally:
            self._unlock_structure()

ScenarioViewState = namedtuple("ScenarioViewState", "scenario name description date")


# Written by Niveditha Kalavakonda (nkalavak@uw.edu)
# Fall 2017 - Dataset Generation

import numpy as np
import matplotlib.pyplot as plt
import cv2
import imutils
import xml.etree.ElementTree as ET
#from shapely.geometry import Polygon, Point
import matplotlib.path as mplPath

# Look-Up Table for Different Segmentation Parts:
# -------------------------------------------------
#    0   Instrument
#   255  Tissue

# Look-Up Table for Different Instrument Types:
# -------------------------------------------------
#   1   Suction
#   2   Electrocautery
#   3   Grasper
#   4   Cutter
#   5   Bipolar

image_path = 'D:/01 Projects/AMAZON CATALYST PROJECT/ColorStats_ToLoad'

save_path = 'D:/01 Projects/AMAZON CATALYST PROJECT/Dataset'

#XML Parsing using ElementTree
xml_path = 'D:/01 Projects/AMAZON CATALYST PROJECT/xml_100instruments'

r_start = 0
c_start = 0
n_rows = 480 #475
n_cols = 720 #520
n_images = 20 #100

background = [0, 0, 0]
instr = [255, 255, 255]
start_image_index = 70 #300

for img in range(start_image_index, start_image_index + n_images):
#img = 14
    index = img #'%03d' % img
    orig_image = cv2.imread(image_path + '/frame' + str(index) + '.jpg')

    #To rotate and flip image

    #new_image1 = cv2.flip(orig_image)
    ##final_image = imutils.rotate_bound(new_image1,90)

    """rows,cols,_ = orig_image.shape
    new_image1 = cv2.flip(orig_image,0)
    M = cv2.getRotationMatrix2D((cols/2,rows/2),-90,1)
    final_image = cv2.warpAffine(new_image1, M, (cols,rows))"""
    #cv2.imshow("Flipped Image", final_image)
    #print (orig_image.shape)
    #print (final_image.shape)

    new_image = np.zeros(orig_image.shape)

    #n_rows,n_cols = n_cols,n_rows

    tree = ET.parse(xml_path + '/frame' + str(index) + '.xml')
    root = tree.getroot()

    x = []
    y = []
    poly1_s = None
    poly1_a = None
    poly2_l = None
    poly3_l = None
    poly4_l = None
    poly5_l = None
    poly6 = None
    poly2_r = None
    poly3_r = None
    poly4_r = None
    poly5_r = None
    poly7 = None
    poly8 = None
    for neighbor in root.iter('object'):
        for charac in neighbor.findall('polygon'):
            points = []
            for verts in charac.iter('pt'):
                y.append(int(verts.find('x').text))
                x.append(int(verts.find('y').text))
            #print (x)
            #print (y)

            points = zip(x,y)
            if(neighbor.find('name').text == 'suction' and (neighbor.find('attributes').text == 'surgeon suction' or neighbor.find('attributes').text == None)):
                poly1_s = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'suction' and neighbor.find('attributes').text == 'assistant suction'):
                poly1_a = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'electrocautery' and (neighbor.find('attributes').text == 'left cautery' or neighbor.find('attributes').text == None)):
                poly2_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'electrocautery' and neighbor.find('attributes').text == 'right cautery'):
                poly2_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'grasper' and (neighbor.find('attributes').text == 'left grasper' or neighbor.find('attributes').text == None)):
                poly3_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'grasper' and neighbor.find('attributes').text == 'right grasper'):
                poly3_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'cutter' and (neighbor.find('attributes').text == 'left cutter' or neighbor.find('attributes').text == None)):
                poly4_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'cutter' and neighbor.find('attributes').text == 'right cutter'):
                poly4_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'pickup' and (neighbor.find('attributes').text == 'left pickup' or neighbor.find('attributes').text == None)):
                poly5_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'pickup' and neighbor.find('attributes').text == 'right pickup'):
                poly5_r = mplPath.Path(list(points))
            if(neighbor.find('name').text ==  'curette'):
                poly6 = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'drill'):
                poly7 = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'others'):
                poly8 = mplPath.Path(list(points))
            """if(neighbor.find('name').text == 'surgeon suction' or neighbor.find('name').text == 'suction'):
                poly1_s = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'assistant suction'):
                poly1_a = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'left cautery' or neighbor.find('name').text == 'electrocautery'):
                poly2_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'right cautery'):
                poly2_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'left grasper' or (neighbor.find('name').text == 'grasper')):
                poly3_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'right grasper'):
                poly3_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'cutter' or (neighbor.find('name').text == 'left cutter')):
                poly4_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'right cutter'):
                poly4_r = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'pickup' or (neighbor.find('name').text == 'left pickup')):
                poly5_l = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'right pickup'):
                poly5_r = mplPath.Path(list(points))
            if(neighbor.find('name').text ==  'curette'):
                poly6 = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'drill'):
                poly7 = mplPath.Path(list(points))
            if(neighbor.find('name').text == 'others'):
                poly8 = mplPath.Path(list(points))"""

            x[:] = []
            y[:] = []
        for c in range(c_start + n_cols):
            for r in range(r_start + n_rows):

                #Check if point is present inside suction mask
                if((poly1_a != None and poly1_a.contains_point((r,c)) == True) or (poly1_s != None and poly1_s.contains_point((r,c)) == True)):
                    new_image[r,c] = instr#orig_image[r,c]

                elif((poly2_l != None and poly2_l.contains_point((r,c)) == True) or (poly2_r != None and poly2_r.contains_point((r,c)) == True)):
                    new_image[r,c] = instr#orig_image[r,c]#instr#

                elif((poly3_l != None and poly3_l.contains_point((r,c)) == True) or (poly3_r != None and poly3_r.contains_point((r,c)) == True)):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                elif((poly4_l != None and poly4_l.contains_point((r,c)) == True) or (poly4_r != None and poly4_r.contains_point((r,c)) == True)):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                elif((poly5_l != None and poly5_l.contains_point((r,c)) == True) or (poly5_r != None and poly5_r.contains_point((r,c)) == True)):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                elif(poly6 != None and poly6.contains_point((r,c)) == True):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                elif(poly7 != None and poly7.contains_point((r,c)) == True):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                elif(poly8 != None and poly8.contains_point((r,c)) == True):
                    new_image[r,c] = instr#orig_image[r,c]#instr

                else:
                    new_image[r,c] = background

    cv2.imwrite(save_path+'/frame'+str(index)+'.jpg',new_image)
    print ("frame "+ str(index) +" : done!")


"""This page is for more details"""
import logging
from typing import List

import streamlit as st

import awesome_streamlit as ast
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import numpy as np

# Get an instance of a logger
logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
def write():
    """Writes content to the app"""
    #ast.shared.components.title_awesome("Detail")      # Titel Awesome_Streamlit

    # Page title
    st.title("Detailed view")

    #-----------------Markdown info CSS-----------------
    st.markdown('''

    ''', unsafe_allow_html=True)

    st.markdown('''

&#x24D8

                Pie Chart

                    The pie chart shows the age distribution worldwide for the selected year.

                Sankey Diagram

                    The Sankey diagram shows the top 10 origin (left) and target (right) countries
                    as well as the distribution of asylum seekers per year.

                Line Chart

                    The line chart shows the development of total annual asylum applications over the years.

        ''', unsafe_allow_html=True)

    # read csv for sankey diagram
    show_df = pd.read_csv('https://raw.githubusercontent.com/hannahkruck/visuasyl/master/src/datasets/Sankey_Diagramm.csv',sep = ';')

    #-----------------Slider-------------------
    # Create Slider and get from above read csv min and max years
    year = st.slider("", (int(show_df["Year"].min())),(int(show_df["Year"].max())))
    # Variable year for both diagrams
    selected_year = year

    #-----------------Page Layout--------------
    # Layout setting of the page
    c1, c2 = st.beta_columns((1, 1))
    container = st.beta_container()
    st.write('', unsafe_allow_html=True)

    #------------------Create Sankey diagram-------------------------------
    # https://www.geeksforgeeks.org/sankey-diagram-using-plotly-in-python/
    # https://coderzcolumn.com/tutorials/data-science/how-to-plot-sankey-diagram-in-python-jupyter-notebook-holoviews-and-plotly#2

    # Variable year for Sankey diagram
    yearVar = selected_year

    # year
    yearRows = show_df[show_df['Year'] != yearVar].index
    show_df.drop(yearRows , inplace=True)

    # Nodes, links, colors
    label_souce = show_df['Label_Source'].dropna(axis=0, how='any')
    label_souce2 = []
    elementVar = ''

    #
    for i in label_souce:
        if(i != elementVar) :
            label_souce2.append(i)
        elementVar = i

    #
    label_target = show_df['Label_Target'].dropna(axis=0, how='any')
    label = [*label_souce2, *label_target]
    source = show_df['Source'].dropna(axis=0, how='any')
    target = show_df['Target'].dropna(axis=0, how='any')
    value = show_df['Value'].dropna(axis=0, how='any')

    # setting color for node and link
    color_node = [
    # Source color order Syria, Afghanistan, Venezuela, Irak, Colombia, Pakistan, Türkei, Nigeria, Iran, Albania
    '#40bf77', '#93beec', '#1ff91f', '#cd8162', '#a6a6a6', '#80e5ff', '#b299e6', '#ff33ff', '#CDC037', '#ff6a6a',
    # Target color order
    '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641', '#0B2641']

    color_link = [
    '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8', '#b8e0b8',
    '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4', '#bed8f4',
    '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be', '#bef4be',
    '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1', '#e7c1b1',
    '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc', '#cccccc',
    '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff', '#80e5ff',
    '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb', '#c2adeb',
    '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff', '#ffccff',
    '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80', '#ffec80',
    '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc', '#ffcccc',]

    # data to dict, dict to sankey
    link = dict(source = source, target = target, value = value, color = color_link)
    node = dict(label = label, pad = 20, thickness = 10, color = color_node)
    layout = dict(
        height = 800,
        font = dict(
             size = 11),
        margin=dict(
            l=0,
            r=0,
            b=0,
            t=30,
        ),
    )

    data = go.Sankey(link = link, node=node)

    # properties sankey diagram layout
    fig2 = go.Figure(data, layout= layout)

    #-----------------Create pie chart-------------------
    # https://jasonxqh.github.io/2020/07/12/plotly基础/

    # read csv for pie chart
    df = pd.read_csv('https://raw.githubusercontent.com/hannahkruck/visuasyl/master/src/datasets/Piechart.csv',sep = ';')

    # show specific value if a choosen year is selected on slider
    labels = df['year'].tolist()
    if selected_year == 2010:
            values = df['2010'].tolist()
    if selected_year == 2011:
            values = df['2011'].tolist()
    if selected_year == 2012:
            values = df['2012'].tolist()
    if selected_year == 2013:
            values = df['2013'].tolist()
    if selected_year == 2014:
            values = df['2014'].tolist()
    if selected_year == 2015:
            values = df['2015'].tolist()
    if selected_year == 2016:
            values = df['2016'].tolist()
    if selected_year == 2017:
            values = df['2017'].tolist()
    if selected_year == 2018:
            values = df['2018'].tolist()
    if selected_year == 2019:
            values = df['2019'].tolist()

    # define color sets
    colors = ['#e6f2ff', '#b3d9ff', '#80bfff', '#3386E6']

    # create pie figure
    fig1 = go.Figure(data=[go.Pie(
            labels = labels,
            values = values,
            insidetextorientation = 'radial',
            hole = 0.399,)])

    # update settings hover and text informations
    fig1.update_traces(
            hoverinfo = 'label+percent+value',
            textinfo = 'percent+label',
            textfont_size = 11,
            marker = dict(colors = colors,
            line = dict(color = 'lightskyblue', width = 0.1)))

    # update settings layout
    fig1.update_layout(dict(
        height = 400,
        font = dict(
            size = 12)))

    # add annotations in the center of the donut pie
    fig1.update_layout(
    annotations=[dict(
            text='Age
 Distribution',
            font_size=12,
            showarrow=False),],
        margin=dict(
            l=0,
            r=0,
            b=0,
            t=50,
        ),
    )
#------------Create Timeline Years V. 2.0-------------------
    # read CSV for the histogram graph
    df = pd.read_csv("https://raw.githubusercontent.com/hannahkruck/visuasyl/master/src/datasets/Histogramm.csv",encoding ="utf8", sep = ";")

    # use years for the x-axis and the worldwide amount of asylum applications for the y-axis
    fig3 = go.Figure(go.Scatter(x = df['year'], y = df['asylum_applications_worldwide']))

    # customizing the graph
    fig3.update_layout(
    # customize height
        height=100,
    # hide labels
        yaxis={'visible': False, 'showticklabels': False
        },
    # show every year as a label below
        xaxis={'type': 'category'},
    # create white background to match with initial background of streamlit
        plot_bgcolor='rgb(255,255,255)',

    # set all margins and padding to zero to create full width graph
        margin=go.layout.Margin(
        l=0,
        r=0,
        b=0,
        t=0,
        pad = 0
    )
)

    with c1:
        st.subheader('Asylum seekers by age in Europe in the year %s' % selected_year)
        st.plotly_chart(fig1, use_container_width=True,config={'modeBarButtonsToRemove': ['lasso2d','select2d', 'pan2d', 'hoverClosestPie']})
    with c2:
        st.subheader('Top 10 Distribution of a Countries Asylum Applications among the various Countries of Destination  %s' % selected_year)
        st.plotly_chart(fig2, use_container_width=True, config={'modeBarButtonsToRemove': ['lasso2d','select2d', 'pan2d', 'hoverClosestCartesian', 'hoverCompareCartesian']})
    with container:
        st.plotly_chart(fig3, use_container_width=True, config=dict(displayModeBar=False))

if __name__ == "__main__":
    write()


# Assign analytes to the right workflow for library QC, based on which prep was
# used.
import sys
import re
from collections import defaultdict
from genologics.lims import *
from genologics import config

DEFAULT_WORKFLOW = "NSC QC"
WORKFLOW_MAPPING = {
        'NSC_16S': 'NSC 16S'
        }

def main(process_id):
    lims = Lims(config.BASEURI, config.USERNAME, config.PASSWORD)
    process = Process(lims, id=process_id)
    analytes = process.all_inputs(unique=True, resolve=True)

    workflow_analytes = defaultdict(list)
    # Identify workflow for each of the samples
    for analyte in analytes:
        sample_prep_used = analyte.samples[0].udf.get('Sample prep NSC')
        workflow = WORKFLOW_MAPPING.get(sample_prep_used, DEFAULT_WORKFLOW)
        workflow_analytes[workflow].append(analyte)

    # Load all workflows in the system
    workflow_data = lims.get_workflows(add_info=True)

    # Look up each workflow name in the list of workflows, and assign it if available
    for workflow_prefix, analytes in workflow_analytes.items():
        for workflow, info in zip(*workflow_data):
            if info['status'] == "ACTIVE" and info['name'].startswith(workflow_prefix):
                lims.route_analytes(analytes, workflow)
                break
        else:
            print (("Error: Unknown workflow '{}' for samples ".format(workflow_prefix) +
                    ", ".join(analyte.name for analyte in analytes) +
                    "."))
            sys.exit(1)

if __name__ == "__main__":
    main(*sys.argv[1:])


#!/usr/bin/python3
# coding: utf-8
## 数据集 MovieLens
# MovieLens 数据集包含多个用户对多部电影的评级数据, 也包括电影元数据信息和用户属性信息; 这个数据集经常用来做推荐系统, 机器学习算法的测试数据集
# 本文所用的数据为 ml-latest-small.zip; 本文主要使用其中 ratings.csv、movies.csv 两个数据表:

## ratings 数据
# userId: 每个用户的 id
# movieId: 每部电影的 id
# rating: 用户评分, 是 5 星制, 按半颗星的规模递增(0.5 stars 5 stars)
# timestamp: 自 1970 年 1 月 1 日零点后到用户提交评价的时间的秒数
# 数据排序的顺序按照 userId, movieId 排列的
# userId,movieId,rating,timestamp
# 1,1,4.0,964982703
# 1,3,4.0,964981247
# 1,6,4.0,964982224
# 1,47,5.0,964983815

## movies 数据
# movieId: 每部电影的 id
# title: 电影的标题
# genres: 电影的类别
# movieId,title,genres
# 1,Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy
# 2,Jumanji (1995),Adventure|Children|Fantasy
# 3,Grumpier Old Men (1995),Comedy|Romance
# 4,Waiting to Exhale (1995),Comedy|Drama|Romance

import math
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
np.random.seed(1024)

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

##################################################################
## 一: 数据处理
moviesPath = '/Users/coder352/datasets/Struct_Data/MovieLens/ml-latest-small/movies.csv'
ratingsPath = '/Users/coder352/datasets/Struct_Data/MovieLens/ml-latest-small/ratings.csv'
moviesDF = pd.read_csv(moviesPath, index_col=None)
ratingsDF = pd.read_csv(ratingsPath, index_col=None)

## 这里我们按照 4:1 的比例将数据集进行拆分, 同时打印出总的用户和电影数量、训练集中的用户和电影数量以及测试集中的用户和电影数量:
trainRatingsDF, testRatingsDF = train_test_split(ratingsDF, test_size=0.2)
print("total_movie_count: " + str(len(set(ratingsDF['movieId'].values.tolist()))))  # total_movie_count: 9724
print("total_user_count: " + str(len(set(ratingsDF['userId'].values.tolist()))))  # total_user_count: 610
print("train_movie_count: " + str(len(set(trainRatingsDF['movieId'].values.tolist()))))  # train_movie_count: 9005
print("test_movie_count: " + str(len(set(testRatingsDF['movieId'].values.tolist()))))  # test_movie_count: 5128
print("train_user_count: " + str(len(set(trainRatingsDF['userId'].values.tolist()))))  # train_user_count: 610
print("test_user_count: " + str(len(set(testRatingsDF['userId'].values.tolist()))))  # test_user_count: 610

## 下面, 使用 pivot_table 得到用户-电影的评分矩阵, 本文中得到 610*8981 的评分矩阵
trainRatingsPivotDF = pd.pivot_table(trainRatingsDF[['userId', 'movieId', 'rating']], columns=['movieId'], index=['userId'], values='rating', fill_value=0)

## 得到电影 id、用户 id 与其索引的映射关系:
# enumerate 返回穷举序列号与值
moviesMap = dict(enumerate(list(trainRatingsPivotDF.columns)))  # 9005 部电影
usersMap = dict(enumerate(list(trainRatingsPivotDF.index)))  # 610 个用户
ratingValues = trainRatingsPivotDF.values.tolist()  # 矩阵变成 list 每一行变成 list 的一个值; 长度为 610 每个值大小为 9005; 意思就是 610 个用户, 每个用户一个 9005 的特征向量
print(len(ratingValues))  # 610
print(len(ratingValues[0]), len(ratingValues[0]))  # 9005 9005

##################################################################
## 二: 相似度计算
## 用户相似度计算
# 利用余弦相似度计算用户之间的相似度
def calCosineSimilarity(list1, list2):
    res = 0
    denominator1 = 0
    denominator2 = 0
    for (val1, val2) in zip(list1, list2):
        res += (val1 * val2)
        denominator1 += val1 ** 2
        denominator2 += val2 ** 2
    return res / (math.sqrt(denominator1 * denominator2))

## 根据上面的相似度函数, 计算用户之间的相似度矩阵(610*610)
# 根据用户对电影的评分, 来判断每个用户间相似度
userSimMatrix = np.zeros((len(ratingValues), len(ratingValues)), dtype=np.float32)
for i in range(len(ratingValues) - 1):  # 这个循环大约 3min
    for j in range(i + 1, len(ratingValues)):
        userSimMatrix[i, j] = calCosineSimilarity(ratingValues[i], ratingValues[j])
        userSimMatrix[j, i] = userSimMatrix[i, j]

## 接下来, 我们要找到与每个用户最相近的 K 个用户, 用这 K 个用户的喜好来对目标用户进行物品推荐, 这里 K=10, 下面的代码用来计算与每个用户最相近的 10 个用户:
# 找到与每个用户最相近的前 K 个用户
userMostSimDict = dict()
for i in range(len(ratingValues)):
    userMostSimDict[i] = sorted(enumerate(list(userSimMatrix[i])), key=lambda x: x[1], reverse=True)[:10]
print(userMostSimDict[0])  # [(265, 0.32571146), (44, 0.31304061), (468, 0.29434624), (312, 0.28957412), (134, 0.28880805), (451, 0.282516), (201, 0.28025714), (367, 0.27581462), (216, 0.27418706), (63, 0.27265775)]

##################################################################
## 三: 推荐电影
# 得到了每个用户对应的 10 个兴趣最相近的用户之后, 我们根据下面的公式计算用户对每个没有观看过的电影的兴趣分:
# 用这 K 个用户的喜好中目标用户没有看过的电影进行推荐
userRecommendValues = np.zeros((len(ratingValues), len(ratingValues[0])), dtype=np.float32)  # 610 * 9005
for i in range(len(ratingValues)):  # 这个循环也挺慢的, 大约 3min
    for j in range(len(ratingValues[i])):
        if ratingValues[i][j] == 0:
            val = 0
            for (user, sim) in userMostSimDict[i]:
                val += (ratingValues[user][j] * sim)  # user 用户对第 j 部电影的打分 * 相似度
            userRecommendValues[i, j] = val
print(userRecommendValues.max())  # 23.9331; 可能最后的分数大于 5

# 接下来为每个用户推荐 10 部电影:
userRecommendDict = dict()
for i in range(len(ratingValues)):
    userRecommendDict[i] = sorted(enumerate(list(userRecommendValues[i])), key=lambda x: x[1], reverse=True)[:10]
print(userRecommendDict[0])  # [(497, 11.047806), (937, 9.8974247), (1185, 9.3699417), (879, 9.3593016), (1034, 9.3171129), (881, 9.312067), (643, 8.8374186), (1610, 8.502533), (837, 7.9373536), (1270, 7.7954793)]

# 将以上代码中使用的索引首先转换为用户 id 与电影 id, 再将电影 id 转换成电影名:
# 将一开始的索引转换为原来用户 id 与电影 id
userRecommendList = []
for key, value in userRecommendDict.items():
    user = usersMap[key]
    for (movieId, val) in value:
        userRecommendList.append([user, moviesMap[movieId]])

# 将推荐结果的电影 id 转换成对应的电影名
recommendDF = pd.DataFrame(userRecommendList, columns=['userId', 'movieId'])
recommendDF = pd.merge(recommendDF, moviesDF[['movieId', 'title']], on='movieId', how='inner')
print(recommendDF.tail(10))

##################################################################
## Tips
# UserCF 在用户特征维度较高(本文有 8981 维)的情况下, 算法效率还是挺低的


import cv2
import numpy as np
from matplotlib import pyplot as plot

def converted(p):
	p[0]=p[0]/2
	p[1]=p[1]*255/100
	p[2]=p[2]*255/100
	return np.array(p)

def imgNeg(image):
	return 255-image

cap=cv2.VideoCapture(0)
while(1):
	#_,raw=cap.read()
	#daw=cv2.Canny(raw,100,250)	#Intensity gradient-Min to max
	#hsv=cv2.cvtColor(raw, cv2.COLOR_BGR2HSV)

	im = cv2.imread('Pictures/Webcam/2015-05-18-035043.jpg')
	imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
	thresh = cv2.threshold(imgray,127,255,0)
	contours= cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
	im = cv2.drawContours(im, contours, -1, (0,255,0), 3)

	#canny=cv2.Canny(hsv,100,250)
	#canny=imgNeg(canny)

	#lower_white=[0,0,65]	#Fine tuned values trained specifically to recognise my tshirt! 145,23,12 also works!
	#upper_white=[360,22,100]
	#lower_blue=[209,30,10]
	#upper_blue=[265,90,97]
	#lower_red=[322,45,5]
	#upper_red=[360,80,90]

	#wmask=cv2.inRange(hsv,converted(lower_white),converted(upper_white))
	#wmask=cv2.medianBlur(wmask,5)
	#wmask=cv2.GaussianBlur(wmask,(5,5),0)

	#bmask=cv2.inRange(hsv,converted(lower_blue),converted(upper_blue))
	#rmask=cv2.inRange(hsv,converted(lower_red),converted(upper_red))
	#rmask=cv2.medianBlur(rmask,5)
	#daw=cv2.Canny(rmask,100,250)
	#whitened_mask=cv2.bitwise_and(raw,raw,mask=wmask)
	#blued_mask=cv2.bitwise_and(raw,raw,mask=bmask)
	#reddened_mask=cv2.bitwise_and(raw,raw,mask=rmask)
	#rmask=cv2.medianBlur(rmask,5)

	#gray=cv2.cvtColor(raw,cv2.COLOR_BGR2GRAY)
	#thresh=cv2.threshold(gray,127,255,0)
	#contours=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
	#raw=cv2.drawContours(raw,contours,-1,(0,255,0),3)

	cv2.imshow('Its such a good vibration!',raw)
	#cv2.imshow('whitened Mask',whitened_mask)
	#cv2.imshow('Canny',daw)
	#cv2.imshow('Reddened Mask',rmask)
	#cv2.imshow('Trackin \'n Crackin\'!',daw)
	#cv2.imshow('Blued Mask',blued_mask)
	#cv2.imshow('My Sketch!',canny)
	k=cv2.waitKey(5) & 0xff
	if k==27:
		break
cv2.destroyAllWindows()


from unittest import mock
from utils.komp2_api import get_allele2gene_map
from utils.models import Allele
import pytest
from sqlalchemy.exc import SQLAlchemyError

@mock.patch('utils.komp2_api.Session')
def test_get_allele2gene_map_one_allele(mock_session):
    test_allele = Allele()
    test_allele.acc = "MGI:4244706"
    test_allele.gf_acc = "MGI:1915571"
    mock_session.return_value.query(Allele).all.return_value = [test_allele]
    allele_gene_map = get_allele2gene_map()
    assert(allele_gene_map["MGI:4244706"] == "MGI:1915571")

@mock.patch('utils.komp2_api.Session')
def test_get_allele2gene_map_no_alleles(mock_session):
    mock_session.return_value.query(Allele).all.return_value = []
    allele_gene_map = get_allele2gene_map()
    assert(allele_gene_map == {})

@mock.patch('utils.komp2_api.Session')
def test_raises(mock_session):
    mock_session.return_value.query(Allele).all.side_effect = lambda: exec('raise(SQLAlchemyError("some info"))')
    with pytest.raises(Exception) as excinfo:
        get_allele2gene_map()
    assert str(excinfo.value) == 'some info'


import sys
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton
from PyQt5.QtWidgets import QLCDNumber, QLabel, QLineEdit
from PyQt5.QtGui import QPixmap

class Example(QWidget):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):

        self.setGeometry(300, 300, 300, 400)
        self.setWindowTitle('Img')

        self.btn = QPushButton('Показать', self)
        self.btn.resize(self.btn.sizeHint())
        self.btn.move(100, 80)
        self.btn.clicked.connect(self.hello)

        # _________input1____________________#
        self.name_label = QLabel(self)
        self.name_label.setText("Имя файла:  ")
        self.name_label.move(40, 50)

        self.name_input = QLineEdit(self)
        self.name_input.move(130, 50)

        self.pix_label = QLabel(self)
        self.pix_label.setText("testtesttesttesttesttesttesttest \n srfgwrfgwerf")
        self.pix_label.setGeometry(40, 150, 200, 200)

    def hello(self):
        a = self.name_input.text()
        pixmap = QPixmap(a)
        self.pix_label.setPixmap(pixmap)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Example()
    ex.show()
    sys.exit(app.exec())


# SJSU CMPE 138Spring2021TEAM8
import simplejson as json
from flask import Flask, make_response, jsonify, Blueprint, request
from flask_mysql_connector import MySQL
from flask import current_app
import random
mysql = MySQL()

animals_api = Blueprint('animals_api', __name__)

@animals_api.route('/animals', methods=['GET', 'POST', 'DELETE', 'PUT'])
def animals():
    if request.method == 'GET':
        try:
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            if 'species_id' in request.args:
                get_animal = """SELECT * FROM Species  \
                          WHERE species_id = %s"""
                cursor.execute(get_animal, (request.args.get('species_id'),))
                current_app.logger.info("Fetching species %s",request.args.get('species_id'))
                rows = cursor.fetchone()
            else:
                get_animal = "SELECT * FROM Species"
                cursor.execute(get_animal)
                current_app.logger.info("Fetching all species in the park")
                rows = cursor.fetchall()
            return make_response(json.dumps(rows), 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'POST':
        try:
            body = request.json
            post_animal = """INSERT INTO Species (species_id,name,age,description,gender,category)
            VALUES ( %s, %s, %s, %s, %s, %s )"""
            species_id =random.randint(100, 999999)
            data = (
                species_id, body['name'], body['age'], body['description'], body['gender'], body['category'])
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(post_animal, data)
            current_app.logger.info("Inserting species %s", body['name'])
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'DELETE':
        try:
            delete_species = """DELETE FROM Species where species_id = %s"""
            data = (request.args.get('species_id'),)
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(delete_species, data)
            current_app.logger.info("Delete species %s", request.args.get('species_id'))
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()
    if request.method == 'PUT':
        try:
            body = request.json
            update_employee = """UPDATE Species set species_id = %s,name = %s,age= %s, description= %s,gender= %s,
            category = %s WHERE species_id = %s"""
            data = (
                body['species_id'], body['name'], body['age'], body['description'], body['gender'], body['category'],
                body['species_id'])
            conn = mysql.connection
            cursor = conn.cursor(dictionary=True)
            cursor.execute(update_employee, data)
            current_app.logger.info("Update species %s", body['name'])
            conn.commit()
            return make_response("true", 200)
        except Exception as e:
            current_app.logger.error(e)
            return make_response("false", 500)
        finally:
            cursor.close()
            conn.close()

@animals_api.route('/animals-stats', methods=['GET'])
def getStats():
    try:
        conn = mysql.connection
        cursor = conn.cursor(dictionary=True, buffered=True)
        stats = {}
        count_query = "SELECT COUNT(*) as total_count FROM Species;"
        cursor.execute(count_query)
        row = cursor.fetchone()
        stats['total_count'] = row['total_count']

        gender_query = """SELECT gender, COUNT(*) as count
                        FROM Species
                        GROUP BY gender;"""
        cursor.execute(gender_query)
        rows = cursor.fetchall()
        stats['gender_stats'] = rows

        category_query = """SELECT category, COUNT(*) as count
                        FROM Species
                        GROUP BY category;"""
        cursor.execute(category_query)
        rows = cursor.fetchall()
        stats['category_stats'] = rows

        age_query = """SELECT SUM(CASE WHEN age < 10 THEN 1 ELSE 0 END) AS 'Under 10',
                        SUM(CASE WHEN age BETWEEN 11 AND 20 THEN 1 ELSE 0 END) AS '11-20',
                        SUM(CASE WHEN age BETWEEN 21 AND 30 THEN 1 ELSE 0 END) AS '21-30',
                        SUM(CASE WHEN age > 30 THEN 1 ELSE 0 END) AS '30 And More'
                        FROM Species"""
        cursor.execute(age_query)
        rows = cursor.fetchall()
        stats['age_stats'] = rows
        return make_response(json.dumps(stats), 200)
    except Exception as e:
        print(e)
        return make_response("false", 500)


import requests
from bs4 import BeautifulSoup
import pandas as pd
import json

import email_sender

COL_title = 'Title'
COL_rating = 'Rating'
COL_review_count = 'Reviews'
COL_taste_profile = 'Taste Profile'
COL_taste_profile_order = 'Taste Profile Order'
COL_image_url_and_shop_url = 'Wine'
COL_image_url_and_shop_url_separator = '||'
COL_price = '$ Normal'
COL_discounted_price = '$ Discount'
COL_final_discount_percent = '% Discount'
COL_offer_type = 'Offer'

def get_number_pages(rode_wijn_page):
    li_wijn_rood = rode_wijn_page.find("li", {"class": "wijn-rood"})
    quantity_span = li_wijn_rood.find('span').find('span')
    quantity_rode_wijn = quantity_span.string[1:-1]
    pages_qty = int(int(quantity_rode_wijn) / 12)
    return pages_qty

def get_wijn_df_columns():
    return [COL_title, COL_rating, COL_review_count, COL_taste_profile,
            COL_image_url_and_shop_url, COL_price, COL_discounted_price,
            COL_final_discount_percent, COL_offer_type]

def new_wijn_df_row(title, rating, review_count, taste_profile, image_url, price, discounted_price,
                    offer_type):

    final_discounted_percent = round(((1 - float(discounted_price) / float(price)) * 100), 2)

    return [title, rating, review_count, taste_profile, image_url, price, discounted_price,
            final_discounted_percent,offer_type]

def parse_wiijn_row_from_article_tag(article):
    img = article.find('img',{'itemprop':'image'})

    title = img['alt']
    rating = article.find('ul',{'class':'rating-box'})['data-rating']
    try:
        h4 = article.find('h4')
        if h4:
            review_count = article.find('h4').contents[0]
        else:
            review_count = 0
    except:
        review_count = -2

    try:
        all_span_title = article.findAll('span',{'class':'title'})
        if len(all_span_title) > 1:
            taste_profile = all_span_title[1].string
        elif len(all_span_title) == 1:
            taste_profile = all_span_title[0].string
        else:
            taste_profile = 'not found'
    except:
        taste_profile = 'error'

    image_url = img['src'] + COL_image_url_and_shop_url_separator + 'https://www.gall.nl' + \
                           article.find('a')['href']
    price = json.loads(article['data-tracking-impression'])['price']

    sticker_title = article.find('div', {'class': 'sticker-title'})
    sticker_text = article.find('div',{'class':'sticker-text'})

    offer_type = ""
    if sticker_title:
        offer_type += str(sticker_title)

    if sticker_text:
        offer_type += str(sticker_text)

    if title.startswith('La Palma Ca') > 0:
        str(1)

    price_item_active = article.find('span', {'class': 'price item active'})
    span_price_discount = article.find('span', {'class': 'price'})

    if price_item_active:
        discounted_price = round(float(price_item_active.text),2)
    elif '2 halen, 1 betalen' in offer_type or '50% korting' in offer_type:
        discounted_price = round(float(price) / 2,2)
    elif '2e halve prijs' in offer_type:
        discounted_price = round(((float(price) / 2) + float(price)) / 2, 2)
    elif span_price_discount:
        discounted_price = round(float(span_price_discount.text), 2)
    else:
        discounted_price = price

    article.find('span',{'class':'price item active'})

    return new_wijn_df_row(
        title,
        rating,
        review_count,
        taste_profile,
        image_url,
        price,
        discounted_price,
        offer_type
    )

def get_wiijn_rows_from_page(rode_wijn_page):
    products_grid = rode_wijn_page.find('div', {'class': 'row products-grid'})
    articles = products_grid.find_all('article')
    wijn_list = []
    for article in articles:
        wijn_list.append(parse_wiijn_row_from_article_tag(article))
    return wijn_list

def get_html_style(table_html):
    return """

            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
            <title>html title</title>

            """ + table_html+''

def get_rode_wijn_df():
    URL_template = 'https://www.gall.nl/shop/acties/?filter=categorie%3Dwijn%2Frode-wijn&page={}'

    wijn_list = []

    url = URL_template.format(1)

    response = requests.get(url)
    rode_wijn_page = BeautifulSoup(response.text, 'html.parser')

    pages_qty = get_number_pages(rode_wijn_page)
    print(str(pages_qty) + ' pages!')
    wiijn_rows = get_wiijn_rows_from_page(rode_wijn_page)
    wijn_list.extend(wiijn_rows)

    for i in range(2, pages_qty + 1):
        url = URL_template.format(i)
        response = requests.get(url)
        rode_wijn_page = BeautifulSoup(response.text, 'html.parser')
        wijn_list.extend(get_wiijn_rows_from_page(rode_wijn_page))
        print('Retrieved page ' + str(i))

    wijn_df = pd.DataFrame(wijn_list, columns=get_wijn_df_columns())
    return wijn_df

def get_rode_wijn_html(wijn_df):
    taste_profile_order = {'Royal Red': 1,
                           'Big Red': 2,
                           'Classic Red': 3,
                           'Easy Rider': 4,
                           'Fresh Red': 5,
                           'Fruit Smoothie': 6}

    wijn_df[COL_taste_profile_order] = wijn_df.apply(
        lambda row: taste_profile_order.setdefault(row[COL_taste_profile], 100), axis=1)
    wijn_df.sort_values(by=[COL_taste_profile_order, COL_rating,COL_review_count,COL_discounted_price], ascending=[True, False,False,True], inplace=True)
    wijn_df_modif_index = wijn_df.set_index([COL_taste_profile, COL_title])

    def format_img(urls):
        urls_array = urls.split(COL_image_url_and_shop_url_separator)
        img_url = urls_array[0]
        shop_url = urls_array[1]
        return ''.format(shop_url, img_url)

    def format_rating(x):
        try:
            return '{0:.2f}'.format(float(x))
        except:
            return '??? ' + str(x)

    columns_to_html = [COL_rating, COL_review_count, COL_discounted_price, COL_price, COL_final_discount_percent,
                       COL_image_url_and_shop_url,COL_offer_type]

    pd.set_option('display.max_colwidth', -1)
    table_html = wijn_df_modif_index.to_html(columns=columns_to_html,
                                       escape=False,
                                       formatters={
                                            COL_image_url_and_shop_url: format_img,
                                            COL_rating: format_rating,
                                            COL_final_discount_percent: lambda x: '{0:.0f}%'.format(x),
                                            COL_discounted_price: lambda x: '${}'.format(x),
                                            COL_price: lambda x: '${}'.format(x),
                                       })

    return get_html_style(table_html)


import matplotlib.pyplot as plt
import pandas as pd
from wordcloud import WordCloud
import ast

def polarity_distribution(df, file_name,title):
    """
    Plot distribution of polarity
    Parameters
    ----------
    df: pd.DataFrame
    file_name: str
    title: str

    Returns
    -------
    None
    """
    save_path = '../reports/figures/'
    if 'sentiment_target' in df.columns:
        plt.figure(figsize=(20,15))
        df_pos = df['sum_polarity_unique_emoji'][df['sentiment_target']=='positive']
        df_neg = df['sum_polarity_unique_emoji'][df['sentiment_target']=='negative']
        plt.hist(df_pos,bins = 20,label='positive')
        plt.hist(df_neg,bins = 20,label='negative')
        plt.title(title,fontsize=25)
        plt.xlabel('Sum of polarity',fontsize=15)
        plt.ylabel('Counts',fontsize=15)
        plt.legend()
    else:
        plt.figure(figsize=(20,15))
        plt.hist(df['sum_polarity_unique_emoji'],bins=40)
        plt.title(title,fontsize=25)
        plt.xlabel('Sum of polarity',fontsize=15)
        plt.ylabel('Counts',fontsize=15)
    plt.savefig(save_path+file_name)

def target_distribution(df,file_name,title):
    """
    Plot target distribution for classes
    Parameters
    ----------
    df: pd.DataFrame
    file_name: str
    title: str

    Returns
    -------
    None
    """
    save_path = '../reports/figures/'
    plt.figure(figsize=(15,10))
    counts = df['sentiment_target'].value_counts()
    plt.bar(counts.index, counts.values)
    plt.title(title,fontsize=25)
    plt.xlabel('Target',fontsize=15)
    plt.ylabel('Counts',fontsize=15)
    plt.savefig(save_path+file_name)

def word_cloud(df,emotion):
    """
    plot word cloud for the most important 100 words
    Parameters
    ----------
    df: pd.DataFrame
    emotion: str

    Returns
    -------

    """
    save_path = '../reports/figures/'
    connect_word_in_tweet = []
    df_neg = df[df['sentiment_target']==emotion]
    for case in df_neg['tokens']:
        x = ast.literal_eval(case)
        x = ' '.join(x)
        connect_word_in_tweet.append(x)
        text = ' '.join(connect_word_in_tweet)
        text = text.replace('-pron-','')
    wordcloud = WordCloud(max_words=100,width=1920, height=1080).generate(text)
    plt.figure( figsize=(20,10) )
    plt.title(emotion)
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.savefig(save_path + 'wordcloud_' + emotion + '.png', facecolor='k', bbox_inches='tight')


# Copyright (C) 2018 Bloomberg LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  <http://www.apache.org/licenses/LICENSE-2.0>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
OperationsService
=================

"""

import logging

import grpc

from google.protobuf.empty_pb2 import Empty

from buildgrid._exceptions import InvalidArgumentError
from buildgrid._protos.google.longrunning import operations_pb2_grpc, operations_pb2
from buildgrid.server._authentication import AuthContext, authorize

class OperationsService(operations_pb2_grpc.OperationsServicer):

    def __init__(self, server):
        self.__logger = logging.getLogger(__name__)

        self._instances = {}

        operations_pb2_grpc.add_OperationsServicer_to_server(self, server)

    # --- Public API ---

    def add_instance(self, instance_name, instance):
        """Registers a new servicer instance.

        Args:
            instance_name (str): The new instance's name.
            instance (OperationsInstance): The new instance itself.
        """
        self._instances[instance_name] = instance

    # --- Public API: Servicer ---

    @authorize(AuthContext)
    def GetOperation(self, request, context):
        self.__logger.debug("GetOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            operation = instance.get_operation(operation_name)
            op = operations_pb2.Operation()
            op.CopyFrom(operation)
            op.name = name
            return op

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return operations_pb2.Operation()

    @authorize(AuthContext)
    def ListOperations(self, request, context):
        self.__logger.debug("ListOperations request from [%s]", context.peer())

        try:
            # The request name should be the collection name
            # In our case, this is just the instance_name
            instance_name = request.name
            instance = self._get_instance(instance_name)

            result = instance.list_operations(request.filter,
                                              request.page_size,
                                              request.page_token)

            for operation in result.operations:
                operation.name = "{}/{}".format(instance_name, operation.name)

            return result

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return operations_pb2.ListOperationsResponse()

    @authorize(AuthContext)
    def DeleteOperation(self, request, context):
        self.__logger.debug("DeleteOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            instance.delete_operation(operation_name)

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return Empty()

    @authorize(AuthContext)
    def CancelOperation(self, request, context):
        self.__logger.debug("CancelOperation request from [%s]", context.peer())

        try:
            name = request.name

            instance_name = self._parse_instance_name(name)
            instance = self._get_instance(instance_name)

            operation_name = self._parse_operation_name(name)
            instance.cancel_operation(operation_name)

        except InvalidArgumentError as e:
            self.__logger.error(e)
            context.set_details(str(e))
            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)

        return Empty()

    # --- Private API ---

    def _parse_instance_name(self, name):
        """ If the instance name is not blank, 'name' will have the form
        {instance_name}/{operation_uuid}. Otherwise, it will just be
        {operation_uuid} """
        names = name.split('/')
        return '/'.join(names[:-1]) if len(names) > 1 else ''

    def _parse_operation_name(self, name):
        names = name.split('/')
        return names[-1] if len(names) > 1 else name

    def _get_instance(self, name):
        try:
            return self._instances[name]

        except KeyError:
            raise InvalidArgumentError("Instance doesn't exist on server: [{}]".format(name))


import json
import dataClasses

def readRaw(filename):
    infile = open(filename,"r")
    indata = infile.read()
    infile.close()
    return indata

def makeEvents(rawdata):
    ou = []
    for x in rawdata["EVENTS"]:
        #x is an event structure
        ev = dataClasses.data_event()
        ev.fromDict(x)
        ou.append(ev)
    return ou

def read(filename):
    raw = readRaw(filename)
    st = json.loads(raw)
    del(raw)
    osc_meta = st["OSC_META"]
    ev_list = makeEvents(st)
    del(st)
    return [osc_meta,ev_list]


import numpy as np
import matplotlib.pyplot as plt

from numpy.fft import fft, fftfreq, ifft

lx = 100 # time in seconds
n = 1000 # number of points

x = np.linspace(1, lx, n) # Time

omg = 2.0*np.pi/lx

y1 = 3.0*np.cos(5.0*omg*x)
y2 = 2.0*np.sin(10.0*omg*x)
y3 = 1.0*np.sin(20.0*omg*x)

y = y3+y2+y1

plt.figure(1)
plt.plot(x, y, '-b')
# plt.show()

ffts = fftfreq(n)
fftvals = fft(y)

#further processing
mask = ffts > 0
ffttheo = 2.0*abs(fftvals/n)

plt.figure(2)
plt.plot(ffts, fftvals, '-g')
# plt.plot(ffts[mask], ffttheo[mask], '-g')
plt.show()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import requests
import lxml
headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36"
    }
url = "http://go.hao123.com"
res= requests.get(url,headers=headers)
html = lxml.etree.HTML(res.text)
tejia =html.xpath("//div[@class='tejia-air-wrapper']")#先找到十个城市
tejias =tejia[0].xpath(".//div[@class='tejia-air']")
file = open("tejia-air.md","w")
for tj in tejias:
    aa = tj.xpath(".//a[@class='flight']")
    for a in aa:
        fromto = a.xpath("./@alog-text")

        price = a.xpath(".//span[@class ='tickets-price']/text()")
        file.write(fromto[0]+price[0]+'    ')
        print(fromto,price)
file.close()


from __future__ import division

import numpy as np
import operator
import scipy.sparse as sp
from numpy.testing import assert_raises
from pyoperators import SparseOperator
from pyoperators.utils.testing import assert_same

A = np.array([[1, 0, 2, 0],
              [0, 0, 3, 0],
              [4, 5, 6, 0],
              [1, 0, 0, 1]])
vecs = [[1, 0, 0, 0],
        [0, 1, 0, 0],
        [0, 0, 1, 0],
        [0, 0, 0, 1]]
formats = 'bsr,csr,csc,coo,dia,dok'.split(',')

def test_matvec():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        so = SparseOperator(cls(A))
        for vec in vecs:
            assert_same(so(vec), np.dot(A, vec))
            assert_same(so.T(vec), np.dot(A.T, vec))
        assert_same(so.todense(), A)
        assert_same(so.T.todense(), A.T)
    for format in formats:
        yield func, format

def test_shapes():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        shapein = (2, 2)
        shapeout = (1, 4, 1)
        so = SparseOperator(cls(A), shapein=shapein, shapeout=shapeout)
        for vec in vecs:
            assert_same(so(np.reshape(vec, shapein)),
                        np.dot(A, vec).reshape(shapeout))
            assert_same(so.T(np.reshape(vec, shapeout)),
                        np.dot(A.T, vec).reshape(shapein))
    for format in formats:
        yield func, format

def test_update_output():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        so = SparseOperator(cls(A))
        out = np.zeros(4, dtype=int)
        outT = np.zeros(4, dtype=int)
        for vec in vecs:
            so(vec, out, operation=operator.iadd)
            so.T(vec, outT, operation=operator.iadd)
        assert_same(out, np.sum(A, axis=1))
        assert_same(outT, np.sum(A, axis=0))
    for format in formats:
        yield func, format

def test_error1():
    values = (sp.lil_matrix(A), np.zeros((10, 4)),
              np.matrix(np.zeros((10, 4))), 3)

    def func(v):
        assert_raises(TypeError, SparseOperator, v)
    for v in values:
        yield func, v

def test_error2():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        sm = cls(A)
        shapein = (2, 3)
        shapeout = (1, 4, 2)
        assert_raises(ValueError, SparseOperator, sm, shapein=shapein)
        assert_raises(ValueError, SparseOperator, sm, shapeout=shapeout)
    for format in formats:
        yield func, format

def test_error3():
    def func(format):
        cls = getattr(sp, format + '_matrix')
        sm = cls(A)
        so = SparseOperator(sm)
        out = np.zeros(4, dtype=int)
        assert_raises(ValueError, so, vecs[0], out, operation=operator.imul)
    for format in formats:
        yield func, format


import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import os, sys
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.dirname(CURRENT_DIR))

from torch.autograd import Variable
from torch.utils.data import TensorDataset,DataLoader,SubsetRandomSampler
from embeddings.model import SNEmbedding,PoincareDistance
from embeddings.rsgd import poincare_grad,RiemannianSGD,euclidean_retraction

class_path = 'class_dict.txt'

split_folder = ('zeroshot/standard_split/AWA2/')
split_version = 1

def get_features():

    train_split_f = open(split_folder+"trainclasses"+str(split_version)+".txt",'r')
    val_split_f = open(split_folder+"valclasses"+str(split_version)+".txt",'r')
    features_f = open("zeroshot/Animals_with_Attributes2/Features/ResNet101/AwA2-features.txt",'r')
    labels_f = open("zeroshot/Animals_with_Attributes2/Features/ResNet101/AwA2-labels.txt",'r')
    class_file = open("zeroshot/Animals_with_Attributes2/classes.txt","r")
    wordnet_file = open("wordnet_dict.txt","r")

    emb = SNEmbedding(1181,32,PoincareDistance,1.0)
    emb.load_state_dict(torch.load('mammals_32d.pth')['model'])
    for i in emb.parameters():
        weights = i

    all_classes = set([])
    train_classes = set([])
    val_classes = set([])
    ci_map = {}

    for line in wordnet_file:
        line = line[:-1].split(",")
        all_classes.add(line[1])
    wordnet_file = open("wordnet_dict.txt","r")

    c = 0
    for line in train_split_f:
        line = line[:-1]
        if line in all_classes:
            train_classes.add(line)
            ci_map[line] = c
            c += 1
    for line in val_split_f:
        line = line[:-1]
        if line in all_classes:
            val_classes.add(line)
            ci_map[line] = c
            c += 1

    class_index_map = {}
    index_class_map = {}
    emb_map = {}
    n = 0
    i_map = {}
    for line in class_file:
        line = line.strip().split("\t")
        if line[1] in train_classes or line[1] in val_classes:
            class_index_map[line[1]] = ci_map[line[1]]
            index_class_map[ci_map[line[1]]] = line[1]
            i_map[n] = ci_map[line[1]]
        n += 1

    for line in wordnet_file:
        line = line[:-1].split(",")
        if int(line[0])-1 in i_map:
            emb_map[class_index_map[line[1]]] = weights[int(line[3])].data

    train_features = []
    train_labels = []
    val_features = []
    val_labels = []

    for feature,label in zip(features_f,labels_f):
        label = int(label)-1
        if label in i_map:
            if i_map[label] in index_class_map.keys():
                if index_class_map[i_map[label]] in train_classes:
                    train_features.append(feature[:-1].split(" "))
                    train_labels.append(i_map[label])
                elif index_class_map[i_map[label]] in val_classes:
                    val_features.append(feature[:-1].split(" "))
                    val_labels.append(i_map[label])

    batch_size = 32
    train_features = np.array(train_features,dtype=np.float32)
    train_labels = np.array(train_labels,dtype=np.float32)
    val_features = np.array(val_features,dtype=np.float32)
    val_labels = np.array(val_labels,dtype=np.float32)
    trainset = TensorDataset(torch.FloatTensor(train_features),torch.LongTensor(train_labels))
    trainloader = DataLoader(trainset,batch_size=batch_size,shuffle=True)
    valset = TensorDataset(torch.FloatTensor(val_features),torch.LongTensor(val_labels))
    valloader = DataLoader(valset,batch_size=batch_size,shuffle=True)

    return trainloader,valloader,index_class_map,emb_map

class ImageEmbedding(nn.Module):
    def __init__(self,input_dim,output_dim,num_classes=39,layers=[],activation=F.relu):
        super(ImageEmbedding, self).__init__()
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.activation = activation
        self.num_classes = num_classes
        self.layers = []
        d = input_dim
        for h_dim in layers:
            self.layers.append(nn.Linear(d,h_dim))
            d = h_dim
        self.layers = nn.ModuleList(self.layers)
        self.output = nn.Linear(d,output_dim)
        self.distfn = PoincareDistance()
        self.softmax = nn.LogSoftmax()

    def forward(self,x,emb_weights):
        for l in self.layers:
            x = l(x)
            x = self.activation(x)
        x = self.output(x)
        x = x/(torch.norm(x,dim=1).reshape([-1,1])+1e-6)
        stacked = torch.stack([x]*emb_weights.shape[1],1)
        dists = self.distfn(stacked,emb_weights)
        return self.softmax(-dists)

def train(model,emb_map,trainloader,valloader,lr=1e-3,n_epochs=5000,n_train=26,n_val=13,batch_size=32):

    train_emb = [torch.FloatTensor(emb_map[i]) for i in range(n_train)]
    train_emb = torch.stack(train_emb)
    val_emb = [torch.FloatTensor(emb_map[i]) for i in range(n_train,n_train+n_val)]
    val_emb = torch.stack(val_emb)

    optimizer = optim.Adam(model.parameters(),lr=lr)
    model_loss = nn.CrossEntropyLoss()

    for ep in range(n_epochs):
        print("Epoch # : " + str(ep))
        num_train = 0.0
        train_loss = 0.0
        train_acc = 0.0

        train_batch_emb = torch.stack([train_emb]*batch_size)
        train_batch_emb = Variable(train_batch_emb,requires_grad=False).cuda()
        for x,y in trainloader:
            x = x.cuda()
            y = y.cuda()
            optimizer.zero_grad()
            if x.shape[0] == batch_size:
                preds = model(x,train_batch_emb)
                loss = model_loss(preds,y)
                _, preds = torch.max(preds.data,1)
                train_acc += torch.sum(preds==y.data)
                train_loss += loss.item()
                num_train += x.shape[0]
                loss.backward(retain_graph=True)
        #             optimizer.step(lr=lr)
                optimizer.step()

        if ep % 1 == 0:
            print("Train loss : " + str(train_loss/num_train))
            print("Train accuracy : " + str(float(train_acc.item())/num_train))

            val_batch_emb = torch.stack([val_emb]*batch_size)
            val_batch_emb = Variable(val_batch_emb,requires_grad=False).cuda()
            num_val = 0.0
            val_loss = 0.0
            val_acc = 0.0
            for x,y in valloader:
                x = x.cuda()
                y = y.cuda()
                y = y-n_train
                if x.shape[0] != batch_size:
                    temp_emb = Variable(torch.stack([val_emb]*x.shape[0]),requires_grad=False).cuda()
                    preds = model(x,temp_emb)
                else:
                    preds = model(x,val_batch_emb)
                loss = model_loss(preds,y)
                val_loss += loss.item()
                num_val += x.shape[0]
                _, preds = torch.max(preds.data,1)
                val_acc += torch.sum(preds==y.data)
            print("Validation loss : " + str(val_loss/num_val))
            print("Validation accuracy : " + str(float(val_acc.item())/num_val))

trainloader,valloader,index_hash,emb_map = get_features()
model = ImageEmbedding(2048,32,layers=[256],num_classes=26).to('cuda')
train(model,emb_map,trainloader,valloader,num_classes=26)


"""empty message

Revision ID: cffecda0acbb
Revises: 70e1cf8d7a57
Create Date: 2019-06-06 10:41:47.728713

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = "cffecda0acbb"
down_revision = "70e1cf8d7a57"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column(
        "users",
        sa.Column(
            "keypair_seed", sa.String(length=128), server_default="Null", nullable=True
        ),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column("users", "keypair_seed")
    # ### end Alembic commands ###


"""views for harshp
"""

from django.http import Http404
from django.shortcuts import render_to_response

from blog.models import BlogPost
from stories.models import StoryPost
from poems.models import Poem
from articles.models import Article
from lifeX.models import LifeXWeek
from brainbank.models import BrainBankIdea
from sitedata.social_meta import create_meta

def home(request):
    """homepage for harshp.com

    show all the latest posts in different apps

    Args:
        request(HttpResponse)

    Returns:
        HttpResponse: 200 on success, 404 on error

    Raises:
        Http404: error retrieving posts
    """
    try:
        blogs = BlogPost.objects.order_by('-published')[:3]
        stories = StoryPost.objects.order_by('-published')[:3]
        poems = Poem.objects.order_by('-published')[:3]
        articles = Article.objects.order_by('-published')[:3]
        brainbank_idea = BrainBankIdea.objects.latest('published')
        lifexweek = LifeXWeek.objects.latest('number')
        description = """
            The personal website of Harshvardhan Pandit (coolharsh55)"""
        keywords = ['harshp.com', 'blog', 'stories', 'poems', ]
        meta = create_meta(
            'harshp.com',
            description,
            keywords,
            url=request.build_absolute_uri(),
        )
    except (BlogPost.DoesNotExist,
            StoryPost.DoesNotExist,
            Poem.DoesNotExist,
            Article.DoesNotExist,
            LifeXWeek.DoesNotExist,
            BrainBankIdea.DoesNotExist):
        raise Http404('Error retrieving website data...')
    return render_to_response(
        'harshp/index.html',
        {
            'blogs': blogs,
            'stories': stories,
            'poems': poems,
            'articles': articles,
            'brainbank_idea': brainbank_idea,
            'lifeXweek': lifexweek,
            'meta': meta,
        }
    )

def changelog(request):
    """changelog

    show the project changelog

    Args:
        request(HttpResponse)

    Returns:
        HttpResponse: 200 on success, 404 on error

    Raises:
        None
    """
    return render_to_response('harshp/changelog.html')

def privacypolicy(request):
    """privacy policy

    show the privacy policy for website

    Args:
        request(HttpResponse)

    Returns:
        HttpResponse: 200 on success, 404 on error

    Raises:
        None
    """
    return render_to_response('harshp/privacypolicy.html')


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

# %% sklearn version: "LDA" algorithm using sklearn library

import numpy as np
from sklearn.discriminant_analysis import \
    LinearDiscriminantAnalysis

import matplotlib.pyplot as plt

# %% part 1 - load dataset and print summary

training_dataset = np.loadtxt('fld.txt', delimiter=',')
X_training_dataset = training_dataset[::, :2]
y_training_dataset = training_dataset[::, 2]

X_class_0_idx = np.where(y_training_dataset == 0)
X_class_1_idx = np.where(y_training_dataset == 1)
X_class_0 = X_training_dataset[X_class_0_idx]
X_class_1 = X_training_dataset[X_class_1_idx]

print('----------------------------------------------------------------------')
print('----------------------------------------------------------------------')
print('Summary of the Dataset:')
print('Class "0" info: Shape - Head - Tail')
print('Shape:', X_class_0.shape)
print('Head:\n', X_class_0[:5, ::])
print('Tail:\n', X_class_0[-5:, ::])
print('----------------------------------------------------------------------')
print('Class "1" info: Shape - Head - Tail')
print('Shape:', X_class_1.shape)
print('Head:\n', X_class_1[:5, ::])
print('Tail:\n', X_class_1[-5:, ::])
print('----------------------------------------------------------------------')

# %% part 2 - perform Linear Discriminant Analysis (LDA)

LDA_object = LinearDiscriminantAnalysis(store_covariance=True)
LDA_object.fit(X_training_dataset, y_training_dataset)
Weight_vector = LDA_object.coef_[0]; intercept = LDA_object.intercept_
print('----------------------------------------------------------------------')
print('LDA >> Slope and Intercept:')
print('Slope (of log-posterior of LDA) =', Weight_vector,
      ',\nIntercept (of log-posterior of LDA) =', intercept)
print('----------------------------------------------------------------------')

pi_0 = len(X_class_0) / (len(X_class_1) + len(X_class_0))
pi_1 = len(X_class_1) / (len(X_class_1) + len(X_class_0))
Cst = np.log(pi_0/pi_1)
# Note >> covariance_array-like of shape (n_features, n_features)
# Note >> means_array-like of shape (n_classes, n_features)
w = np.dot(np.linalg.inv(LDA_object.covariance_),
           (LDA_object.means_[0] - LDA_object.means_[1]))
DC_term = Cst - 0.5 * np.dot((LDA_object.means_[0] + LDA_object.means_[1]).T, w)

# %% part 3 - prediction

predictions = (np.sign(np.dot(w, X_training_dataset.T) + DC_term) + 1) / 2
error_possibility_1 = sum(predictions != y_training_dataset)
error_possibility_2 = sum((1 - predictions) != y_training_dataset)
rel_error_1 = error_possibility_1 / len(y_training_dataset)
rel_error_2 = error_possibility_2 / len(y_training_dataset)

if rel_error_1 < rel_error_2:
    final_predictions = predictions
else:
    final_predictions = 1 - predictions

num_preds_to_print = 20
print(f'Some of predictions are: [first {num_preds_to_print}]\n',
      final_predictions[:num_preds_to_print])
print(f'Some of predictions are: [last {num_preds_to_print}]\n',
      final_predictions[-num_preds_to_print:])

# %% part 4 - error report

errorIndex = np.argwhere(final_predictions != y_training_dataset)
errorPts = X_training_dataset[errorIndex]
errorPts = np.squeeze(errorPts)

print('----------------------------------------------------------------------')
print('LDA >> Error:', 100 * min(rel_error_2, rel_error_1), '%.')
print('LDA >> LDA_object.score():',
      100 * LDA_object.score(X_training_dataset, y_training_dataset), '%.')
print('----------------------------------------------------------------------')
print('----------------------------------------------------------------------')

# %% part 5 - visualization

#  first plot
figure_width = 20
original_data_linewidth = 5
legend_fontsize = 20
plot_grid_option = False

plt.figure(figsize=(figure_width, figure_width / 1.618))
plt.scatter(X_class_0[:, 0],
            X_class_0[:, 1],
            c='r', marker='.',
            linewidths=original_data_linewidth)
plt.scatter(X_class_1[:, 0],
            X_class_1[:, 1],
            c='b', marker='.',
            linewidths=original_data_linewidth)

k0, k1 = 5, 3
plt.plot([- k0 * w[0], k1 * w[0]],
         [-k0 * w[1], k1 * w[1]],
         'g--', lw=5)

plt.xlabel('first axis (x0)', size=legend_fontsize)
plt.ylabel('second axis (x1)', size=legend_fontsize)
plt.legend(['LDA line', 'original data - class 0', 'original data - class 1'],
           fontsize=legend_fontsize)

plt.savefig('sklearn-improved-img-1.png', dpi=300)
plt.grid(True)
plt.savefig('sklearn-improved-img-1-grid.png', dpi=300)
plt.show()

#  second plot
plt.figure(figsize=(figure_width, figure_width / 1.618))
plt.scatter(X_class_0[:, 0],
            X_class_0[:, 1],
            c='r', marker='.',
            linewidths=original_data_linewidth)
plt.scatter(X_class_1[:, 0],
            X_class_1[:, 1],
            c='b', marker='.',
            linewidths=original_data_linewidth)

k0, k1 = 5, 3
plt.plot([- k0 * w[0], k1 * w[0]],
         [-k0 * w[1], k1 * w[1]],
         'g--', lw=5)

plt.scatter(errorPts[:, 0],
            errorPts[:, 1],
            c='orange',
            marker='o')

plt.xlabel('first axis (x0)', size=legend_fontsize)
plt.ylabel('second axis (x1)', size=legend_fontsize)
plt.legend(['LDA line',
            'original data - class 0',
            'original data - class 1',
            'LDA error samples'],
           fontsize=legend_fontsize)

plt.savefig('sklearn-improved-img-2.png', dpi=300)
plt.grid(True)
plt.savefig('sklearn-improved-img-2-grid.png', dpi=300)
plt.show()


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.9-x86_64/egg/dreegdl/smote.py
# Compiled at: 2020-03-04 18:08:43
# Size of source mod 2**32: 2485 bytes
from sklearn.neighbors import NearestNeighbors
import random as rd

class Smote:
    __doc__ = '\n  Implement SMOTE, synthetic minority oversampling technique.\n  Parameters\n  -----------\n  sample      2D (numpy)array\n              minority class samples\n  N           Integer\n              amount of SMOTE N%\n  k           Integer\n              number of nearest neighbors k\n              k <= number of minority class samples\n  Attributes\n  ----------\n  newIndex    Integer\n              keep a count of number of synthetic samples\n              initialize as 0\n  synthetic   2D array\n              array for synthetic samples\n  neighbors   K-Nearest Neighbors model\n  '

    def __init__(self, sample, N, k):
        self.sample = sample
        self.k = k
        self.T = len(self.sample)
        self.N = N
        self.newIndex = 0
        self.synthetic = []
        self.neighbors = NearestNeighbors(n_neighbors=(self.k)).fit(self.sample)

    def over_sampling(self):
        if self.N < 100:
            self.T = self.N / 100 * self.T
            self.N = 100
        self.N = int(self.N / 100)
        for i in range(0, self.T):
            nn_array = self.compute_k_nearest(i)
            self.populate(self.N, i, nn_array)

    def compute_k_nearest(self, i):
        nn_array = self.neighbors.kneighbors([self.sample[i]], (self.k), return_distance=False)
        if len(nn_array) is 1:
            return nn_array[0]
        return []

    def populate(self, N, i, nn_array):
        while N is not 0:
            nn = rd.randint(1, self.k - 1)
            self.synthetic.append([])
            for attr in range(0, len(self.sample[i])):
                dif = self.sample[nn_array[nn]][attr] - self.sample[i][attr]
                gap = rd.random()
                while gap == 0:
                    gap = rd.random()

                self.synthetic[self.newIndex].append(self.sample[i][attr] + gap * dif)

            self.newIndex += 1
            N -= 1


from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.serializers import ValidationError

from ..assessment.api import (
    AssessmentEditViewset,
    AssessmentLevelPermissions,
    AssessmentViewset,
    DisabledPagination,
    InAssessmentFilter,
    get_assessment_id_param,
)
from ..assessment.models import Assessment, TimeSpentEditing
from ..common.api import CleanupFieldsBaseViewSet, LegacyAssessmentAdapterMixin
from ..common.helper import re_digits, tryParseInt
from ..common.renderers import PandasRenderers
from ..common.serializers import UnusedSerializer
from ..common.views import AssessmentPermissionsMixin
from ..mgmt.models import Task
from ..riskofbias import exports
from ..study.models import Study
from . import models, serializers

class RiskOfBiasAssessmentViewset(
    AssessmentPermissionsMixin, LegacyAssessmentAdapterMixin, viewsets.GenericViewSet
):
    parent_model = Assessment
    model = Study
    permission_classes = (AssessmentLevelPermissions,)
    serializer_class = UnusedSerializer
    lookup_value_regex = re_digits

    def get_queryset(self):

        perms = self.get_obj_perms()
        if not perms["edit"]:
            return self.model.objects.published(self.assessment)
        return self.model.objects.get_qs(self.assessment.id)

    @action(detail=True, methods=("get",), url_path="export", renderer_classes=PandasRenderers)
    def export(self, request, pk):
        self.set_legacy_attr(pk)
        self.permission_check_user_can_view()
        rob_name = self.assessment.get_rob_name_display().lower()
        exporter = exports.RiskOfBiasFlat(
            self.get_queryset(), filename=f"{self.assessment}-{rob_name}"
        )

        return Response(exporter.build_export())

    @action(detail=True, methods=("get",), url_path="full-export", renderer_classes=PandasRenderers)
    def full_export(self, request, pk):
        self.set_legacy_attr(pk)
        self.permission_check_user_can_view()
        rob_name = self.assessment.get_rob_name_display().lower()
        exporter = exports.RiskOfBiasCompleteFlat(
            self.get_queryset(), filename=f"{self.assessment}-{rob_name}-complete"
        )
        return Response(exporter.build_export())

class RiskOfBiasDomain(viewsets.ReadOnlyModelViewSet):
    assessment_filter_args = "assessment"
    model = models.RiskOfBiasDomain
    pagination_class = DisabledPagination
    permission_classes = (AssessmentLevelPermissions,)
    filter_backends = (InAssessmentFilter, DjangoFilterBackend)
    serializer_class = serializers.AssessmentDomainSerializer
    lookup_value_regex = re_digits

    def get_queryset(self):
        return self.model.objects.all().prefetch_related("metrics")

class RiskOfBias(viewsets.ModelViewSet):
    assessment_filter_args = "study__assessment"
    model = models.RiskOfBias
    pagination_class = DisabledPagination
    permission_classes = (AssessmentLevelPermissions,)
    filter_backends = (InAssessmentFilter, DjangoFilterBackend)
    serializer_class = serializers.RiskOfBiasSerializer
    lookup_value_regex = re_digits

    def get_queryset(self):
        return self.model.objects.all().prefetch_related(
            "study", "author", "scores__metric__domain"
        )

    def perform_update(self, serializer):
        super().perform_update(serializer)
        study = serializer.instance.study
        user = self.request.user
        Task.objects.ensure_rob_started(study, user)
        if serializer.instance.final and serializer.instance.is_complete:
            Task.objects.ensure_rob_stopped(study)

        # send time complete task
        if not serializer.errors:
            TimeSpentEditing.add_time_spent_job(
                self.request.session.session_key,
                serializer.instance.get_edit_url(),
                serializer.instance,
                serializer.instance.get_assessment().id,
            )

    def create(self, request, *args, **kwargs):
        study_id = tryParseInt(request.data.get("study_id"), -1)

        try:
            study = Study.objects.get(id=study_id)
        except ObjectDoesNotExist:
            raise ValidationError("Invalid study_id")

        # permission check using the user submitting the request
        if not study.user_can_edit_study(study.assessment, request.user):
            raise PermissionDenied(
                f"Submitter '{request.user}' has invalid permissions to edit Risk of Bias for this study"
            )

        # overridden_objects is not marked as optional in RiskOfBiasScoreSerializerSlim; if it's not present
        # in the payload, let's just add an empty array.
        scores = request.data.get("scores")
        for score in scores:
            if "overridden_objects" not in score:
                score["overridden_objects"] = []

        return super().create(request, *args, **kwargs)

    @action(detail=True, methods=["get"])
    def override_options(self, request, pk=None):
        object_ = self.get_object()
        return Response(object_.get_override_options())

class AssessmentMetricViewset(AssessmentViewset):
    model = models.RiskOfBiasMetric
    serializer_class = serializers.AssessmentMetricChoiceSerializer
    pagination_class = DisabledPagination
    assessment_filter_args = "domain__assessment"

    def get_queryset(self):
        return self.model.objects.all()

class AssessmentMetricScoreViewset(AssessmentViewset):
    model = models.RiskOfBiasMetric
    serializer_class = serializers.AssessmentMetricScoreSerializer
    pagination_class = DisabledPagination
    assessment_filter_args = "domain__assessment"

    def get_queryset(self):
        return self.model.objects.all()

class AssessmentScoreViewset(AssessmentEditViewset):
    model = models.RiskOfBiasScore
    pagination_class = DisabledPagination
    assessment_filter_args = "metric__domain_assessment"
    serializer_class = serializers.RiskOfBiasScoreSerializer

    def get_assessment(self, request, *args, **kwargs):
        assessment_id = get_assessment_id_param(request)
        return get_object_or_404(self.parent_model, pk=assessment_id)

    @action(detail=False)
    def choices(self, request):
        assessment_id = self.get_assessment(request)
        rob_assessment = models.RiskOfBiasAssessment.objects.get(assessment_id=assessment_id)
        return Response(rob_assessment.get_rob_response_values())

    def create(self, request, *args, **kwargs):
        # create using one serializer; return using a different one
        serializer = serializers.RiskOfBiasScoreOverrideCreateSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)
        new_serializer = serializers.RiskOfBiasScoreSerializer(serializer.instance)
        headers = self.get_success_headers(new_serializer.data)
        return Response(new_serializer.data, status=status.HTTP_201_CREATED, headers=headers)

    def perform_destroy(self, instance):
        if instance.is_default:
            raise PermissionDenied("Cannot delete a default risk of bias score")
        instance.delete()

class ScoreCleanupViewset(CleanupFieldsBaseViewSet):
    model = models.RiskOfBiasScore
    serializer_class = serializers.RiskOfBiasScoreCleanupSerializer
    assessment_filter_args = "metric__domain__assessment"