# -*- coding: utf-8 -*-
"""
Created on Sat Aug 29 00:34:41 2020

@author: IŞIK
"""

#%%
#Kütüphaneler

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.impute import SimpleImputer
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression

#%%
#Verilerin Okunması

veriler = pd.read_csv("eksikveriler.csv")

ulkeler = veriler.iloc[:,0:1].values
boy_kilo_yas = veriler.iloc[:,1:4].values
cinsiyet = veriler[['cinsiyet']]
boy = veriler.iloc[:,1:2]

#%%
#Verilerin Ön İşlemesi

imputer = SimpleImputer(missing_values = np.nan, strategy = "mean")
boy_kilo_yas = imputer.fit_transform(boy_kilo_yas)

##Kategorize Edildi
le = preprocessing.LabelEncoder()
ohe = preprocessing.OneHotEncoder()

ulkeler = ohe.fit_transform(ulkeler).toarray()
cinsiyet = ohe.fit_transform(cinsiyet).toarray()

##Dataframe Dönüştürüldü
ulkeler = pd.DataFrame(data = ulkeler, index = range(22), columns = ["fr", "tr", "us"])
cinsiyet = pd.DataFrame(data = cinsiyet[:,0:1], index = range(22), columns = ["cinsiyet"])
boy_kilo_yas = pd.DataFrame(data = boy_kilo_yas, index = range(22), columns = ["boy", "kilo", "yas"])

##DataFrameler birleştirildi
sonuc = pd.concat([ulkeler, boy_kilo_yas], axis = 1)
sonuc = pd.concat([sonuc, cinsiyet], axis = 1)

ulke_boy_kilo_yas = pd.concat([ulkeler, boy_kilo_yas], axis = 1)
ulke_kilo_yas_cinsiyet = pd.concat([sonuc.iloc[:,0:3], sonuc.iloc[:,4:]], axis = 1)

##Eğitilmesi (Train Test)
x_train, x_test, y_train, y_test = train_test_split(ulke_boy_kilo_yas, cinsiyet, test_size = 0.33, random_state = 0)

##StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(x_train)
X_test =sc.fit_transform(x_test)
'''
##StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(x_train)
X_test = sc.fit_transform(x_test)
Y_train = sc.fit_transform(y_train)
Y_test = sc.fit_transform(y_test)
'''
lr = LinearRegression()
lr.fit(x_train, y_train)

##tahmin
tahmin = lr.predict(x_test)

##Boyun bulunması
lr2 = LinearRegression()
x_train, x_test, y_train, y_test = train_test_split(ulke_kilo_yas_cinsiyet, boy, test_size = 0.33, random_state = 0)

lr2.fit(x_train, y_train)
tahmin2 = lr2.predict(x_test)

#%%
#Görselleştirme
x_train = x_train.sort_index()      #İndexleri random bir şekilde olan dataframe sıralı hhale dönüştürür
y_train = y_train.sort_index()

plt.plot(x_train, y_train)
plt.plot(x_test, tahmin)


import cv2
import argparse
import os

def get_args():
    parser = argparse.ArgumentParser(description='Capture image.')
    parser.add_argument('--camera_index', type=int, default=0, help='Camera index.')
    parser.add_argument('--save_dir', type=str, default='./data/images/', help='Save dir')
    parser.add_argument('--width', type=int, help='Video width.')
    parser.add_argument('--height', type=int, help='Video height.')

    return parser.parse_args()

if __name__ == '__main__':

    args = get_args()

    assert(len(args.save_dir) > 0)

    cap = cv2.VideoCapture(args.camera_index)

    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) if args.width == None else args.width
    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) if args.height == None else args.height

    if not os.path.exists(args.save_dir):
        os.mkdir(args.save_dir)

    idx = 0
    while True:
        ret, frame = cap.read()
        cv2.imshow('Capture', frame)

        if cv2.waitKey(1) & 0xFF == ord('s'):
            file_name = args.save_dir + str(idx) + '.png'
            cv2.imwrite(file_name, frame)
            print('Saved image to ', file_name)
            idx += 1

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cap.release()


from typing import List

class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class PrintBinaryTree:
    def print_tree(self, root: TreeNode) -> List[List[str]]:
        # Get the height of the tree
        self.height = self.get_height(root)
        print(self.height)
        d = 2 ** self.height - 1
        # build the matrix
        result = [["" for i in range(d)] for j in range(self.height)]
        # draw on the matrix
        self.draw(root, result, self.height, d // 2)
        return result

    def draw(self, node, result, height, position):
        # draw the node
        result[self.height - height][position] = str(node.val)

        # if this is the lowest level, we are done
        if height == 1:
            return

        # calculate the depth of the current node
        node_depth = self.height - height + 1

        # how much away should we place the child node(s)
        child_sep = 2 ** (self.height - node_depth - 1)

        # recursively draw the children node (DFS)
        if node.left is not None:
            child_pos = position - child_sep
            self.draw(node.left, result, height - 1, child_pos)
        if node.right is not None:
            child_pos = position + child_sep
            self.draw(node.right, result, height - 1, child_pos)

    def get_height(self, node):
        if node.left is None and node.right is None:
            return 1

        if node.left is None and node.right is not None:
            return 1 + self.get_height(node.right)

        if node.left is not None and node.right is None:
            return 1 + self.get_height(node.left)

        if node.left is not None and node.right is not None:
            return 1 + max(self.get_height(node.left), self.get_height(node.right))


from discord.errors import DiscordException
from ..__imports__ import *
from ..settings import *
from .discord_init import DiscordInit

class YTDLSource(discord.PCMVolumeTransformer):
    def __init__(self, source, *, data, volume=0.5, ytdl_format_options=ytdl_format_options):
        super().__init__(source, volume)
        youtube_dl.utils.bug_reports_message = lambda: ''
        self.ytdl = youtube_dl.YoutubeDL(ytdl_format_options)
        self.data = data
        self.title = data.get('title')
        self.url = data.get('url')

    @classmethod
    async def from_url(self, cls, url, *, loop=None, stream=False):
        loop = loop or asyncio.get_event_loop()
        data = await loop.run_in_executor(None, lambda: self.ytdl.extract_info(url, download=not stream))

        if 'entries' in data:
            # take first item from a playlist
            data = data['entries'][0]

        filename = data['url'] if stream else self.ytdl.prepare_filename(data)
        return cls(discord.FFmpegPCMAudio(filename, **ffmpeg_options), data=data)

class MusicMixin(DiscordInit, commands.Cog):
    StartTime = 0
    lastPod = None

    @commands.command()
    async def join(self, ctx, *, channel: discord.VoiceChannel):
        if ctx.voice_client is not None:
            return await ctx.voice_client.move_to(channel)
        await channel.connect()

    @commands.command(pass_context=True, aliases=['oldp', 'olds'])
    async def play(self, ctx, *, url="https://youtu.be/dQw4w9WgXcQ"):

        # DISABLED
        return
        #await ctx.message.add_reaction('🎧')
        if ("youtube.com" in str(url) or "youtu.be"):
            async with ctx.typing():
                player = await YTDLSource.from_url(url=url, loop=self.client.loop, stream=True)
                ctx.voice_client.play(player, after=None)
                if(str(url) == "https://youtu.be/dQw4w9WgXcQ"):
                    embed = discord.Embed(title="You need to give a url!", colour=discord.Colour(0xff5065), url=url, description=player.title)
                    embed.set_image(url="https://i.imgur.com/xrBXtFh.png")
                else:
                    embed = discord.Embed(title="Playing from Youtube", colour=discord.Colour(0xff5065), url=url, description=player.title)
                    y = re.search("/?v=(.{,11})", url).groups()[0]
                    try:
                        embed.set_image(url=f"https://img.youtube.com/vi/{y}/0.jpg")
                    except:
                        pass
        else:
            embed = discord.Embed(title=f"Searching : {str(url)}", colour=discord.Colour(0xff5065))
            async with ctx.typing():
                player = await YTDLSource.from_url(url, loop=self.client.loop, stream=True)
                ctx.voice_client.play(player, after=None)

        embed.set_author(name=ctx.message.author.name,icon_url=ctx.message.author.avatar_url)
        embed.set_footer(text=self.name, icon_url=self.avatar)
        await ctx.reply(embed=embed)

    @commands.command(pass_context=True, aliases=['oldpl'])
    async def lofi(self, ctx, *, url="https://youtu.be/5qap5aO4i9A"):
        # DISABLED
        return
        #await ctx.message.add_reaction('🎧')
        async with ctx.typing():
            player = await YTDLSource.from_url(url, loop=self.client.loop, stream=True)
            ctx.voice_client.play(player, after=None)
            embed = discord.Embed(title="Playing from Youtube", colour=discord.Colour(0xff5065), url=url, description=player.title)
            embed.set_image(url="https://i.ytimg.com/vi/5qap5aO4i9A/maxresdefault.jpg")
        embed.set_author(name=ctx.message.author.name,icon_url=ctx.message.author.avatar_url)
        embed.set_footer(text=self.name, icon_url=self.avatar)
        await ctx.reply(embed=embed)

    @commands.command(aliases=['oldpodp'])
    async def podplay(self,ctx,epno=0):

        # DISABLED
        return

        podepi = epno
        if(self.lastPod == None):
            embed = discord.Embed(colour=discord.Colour(
                0xbd10e0), description=" ")
            embed.set_thumbnail(url=self.avatar)
            embed.set_author(name=self.name, url=self.avatar,icon_url=self.avatar)
            embed.add_field(name=f"No Recent Podcast Searches",value=f"search for podcast using {self.pre}pod",inline=False)
            embed.set_thumbnail(url=self.avatar)
            await ctx.send(embed=embed)
        else:
            currentpod = self.lastPod
            try:
                await self.playPodcast(ctx,podepi=podepi,currentpod=currentpod)
                embed = discord.Embed(title=currentpod.GetEpisodeDetails(podepi)['title'],
                                      colour=discord.Colour(0xb8e986), url=currentpod.GetEpisodeDetails(podepi)['link'],
                                      description=currentpod.GetEpisodeDetails(podepi)['summsary'],
                                      inline=False)
                embed.set_thumbnail(url=currentpod.PodcastImage(podepi))
                embed.set_author(name=self.name,icon_url=self.avatar)
                embed.set_footer(text=currentpod.GetEpisodeDetails(podepi)['title'],icon_url=self.avatar)
                await ctx.send(embed=embed)
            except AttributeError:
                await ctx.send("You aren't in voice channel m8")

    @commands.command(aliases=['oldpodcast'])
    async def pod(self,ctx , * , strparse = " "):

        # DISABLED
        await ctx.message.add_reaction('❕')
        await ctx.message.reply("This feature is currently disabled")
        return

        if(':' in strparse):
            podname_,num = strparse.replace(' ','').split(':')
            podepi = int(num)
        elif(not strparse == ' '):
            podname_ = strparse.replace(' ','')
            podepi = None
        else:
            podname_ = ' '
            podepi = None

        try:
            start = int(podname_.split('-')[1])
            podname = podname_.split('-')[0]
        except:
            start = 0
            podname = podname_
        if(podname == " "):
            embed = discord.Embed(colour=discord.Colour(0x91ff), description="Podcast Section")
            embed.set_thumbnail(url=self.avatar)
            embed.set_author(name=self.name, icon_url=self.avatar)
            embed.add_field(name=f"{self.pre}pod", value="This very command you ran",inline=False)
            embed.add_field(name=f"{self.pre}pod [Name of Podcast]",value="Searches for the Podcast and shows Episodes related to it.", inline=False)
            embed.add_field(name=f"{self.pre}pod [Name of Podcast] : [Selection Number] or {self.pre}podp [Selection No]", value="Play the podcast selection , default 0 plays the latest available episode",inline=False)
            embed.add_field(name=f"{self.pre}stop or {self.pre}dc",value="Stop and Disconnect\n Sadly Haven't Implemented any Pause for now", inline=False)
        if(podepi == None and not podname == " "):
            await ctx.send(f'Searching 🔍')
            embed = discord.Embed(colour=discord.Colour(
                0xbd10e0), description=" ")
            embed.set_thumbnail(url=self.avatar)
            embed.set_author(name=self.name, url=self.avatar,
                             icon_url=self.avatar)
            try:
                k = ph.PodSearch(podname)
            except json.JSONDecodeError:
                embed.add_field(name=f"Corrupted Feed",value="Command raised an exception: JSONDecodeError",inline=False)
                return
            except:
                embed.add_field(name=f"Somewhere Something went wrong",
                                value=r"I have 0 clue what the hell happened rn ¯\_(ツ)_/¯", inline=False)
                return
            await ctx.message.add_reaction('⏳')
            if(not k['name'] == "Podcast Not Found"):
                currentpod = ph.Podcast(k['name'], k['rss'])
                self.lastPod = currentpod
                paginationsize = ph.Pagination(k['count'],5)
                ind = 0 + 5*start
                for episode_ in currentpod.ListEpisodes()[start:start+5]:
                    embed.add_field(name=f"{ind} : "+episode_,value=k['date'],inline=False)
                    ind += 1
                if(paginationsize > 1):
                    embed.add_field(name="Change Page",value=f"```{self.pre}pod {podname} - [Page_Number]```")
                embed.set_footer(text=f"Page {start}/{paginationsize}", icon_url=self.avatar)
                try:
                    embed.set_thumbnail(url=k['image'])
                except:
                    embed.set_thumbnail(url=self.avatar)
            else:
                embed.add_field(name=f"No Podcasts Found",value="itunes returned no results",inline=False)
                embed.set_thumbnail(url=self.avatar)

            embed.set_thumbnail(url=currentpod.PodcastImage(podepi))
            embed.set_author(name=self.name,icon_url=self.avatar)
            embed.set_footer(text=k['name'],icon_url=self.avatar)

        await ctx.send(embed=embed)

    async def playPodcast(self, context, podepi, currentpod):
        try:
            await context.voice_client.disconnect()
        except:
            pass
        if context.voice_client is None:
            if context.author.voice.channel:
                await context.author.voice.channel.connect()

        guild = context.guild
        voice_client: discord.VoiceClient = discord.utils.get(self.client.voice_clients, guild=guild)
        _source_ = currentpod.GetEpisodeMp3(podepi)
        audio_source = discord.FFmpegPCMAudio(_source_)
        if not voice_client.is_playing():
            voice_client.play(audio_source, after=None)

    @commands.command(aliases=['oldfuckoff', 'olddc' , 'olddisconnect'])
    async def stop(self, ctx ):
        if(ctx.author.voice.channel):
            await ctx.message.add_reaction('👍')
            embed = discord.Embed( title=f"Exiting", description=f"played" ,colour=discord.Colour(0xff5065))
            embed.set_author(name=ctx.message.author.name,icon_url=ctx.message.author.avatar_url)
            embed.set_footer(text=self.client.user.name,icon_url=self.client.user.avatar_url)
            await ctx.voice_client.disconnect()
            return await ctx.reply(embed=embed)
        else:
            await ctx.message.add_reaction('❗')
            embed = discord.Embed( title=f"you are not in the voice channel", colour=discord.Colour(0xff5065))
            embed.set_author(name=ctx.message.author.name,icon_url=ctx.message.author.avatar_url)
            embed.set_footer(text=self.client.user.name,icon_url=self.client.user.avatar_url)
            return await ctx.reply(embed=embed)

    @lofi.before_invoke
    @play.before_invoke
    async def ensure_voice(self, ctx):

        # DISABLED
        await ctx.message.add_reaction('❕')
        await ctx.message.reply("This feature is currently disabled")
        return

        if ctx.voice_client is None:
            if ctx.author.voice.channel:
                self.StartTime = ttime.time()
                print("timer started")
                await ctx.author.voice.channel.connect()
            else:
                embed = discord.Embed(title=f"{ctx.message.author.mention} is not connected to any Voice channel", colour=discord.Colour(0xff5065))
                embed.set_author(name=ctx.message.author.name,icon_url=ctx.message.author.avatar_url)
                embed.set_footer(text=self.client.user.name,icon_url=self.client.user.avatar_url)
                return await ctx.send(embed=embed)

        elif ctx.voice_client.is_playing():
            ctx.voice_client.stop()

def setup(bot):
    bot.add_cog(MusicMixin(bot))


#!/usr/bin/python

def outlierCleaner(data):
    """
        Clean away the 10% of points that have the largest
        residual errors (difference between the prediction
        and the actual net worth).

        Return a list of tuples named cleaned_data where
        each tuple is of the form (age, net_worth, error).
    """

    print("hello")
    cleaned_data = []

    data[3]=0
    ### your code goes here
    from scipy.spatial import distance
    import pandas as pd
    print(type(data))
    for i in range(0, len(data.index)):
        a = (data[1][i] , data[0][i])
        b = (data[1][i], data[2][i])
        data[3][i] =  distance.euclidean(a,b)

    print(data)
    sdata = data.sort(columns=3, ascending=False)
    sdata = sdata.reset_index()
    cl_data = sdata[int(len(sdata)*0.1):]

    cl_data = cl_data.loc[:,[1,2,3]]

    return cl_data
    #return cleaned_data


# -*- coding: utf-8 -*-
from config import db

class SpiderDeploy(db.Model):
    __tablename__ = "spider_deploy"

    deploy_id = db.Column(db.Integer, primary_key=True, nullable=False)
    description = db.Column(db.String(30))
    deployed_at = db.Column(db.DateTime)
    created_at = db.Column(db.DateTime)
    updated_at = db.Column(db.DateTime)

    def __init__(self, description,deployed_at,created_at,updated_at):
        '''Constructor'''
        self.description=description
        self.deployed_at=deployed_at
        self.created_at=created_at
        self.updated_at=updated_at

    def __repr__(self):
        return 'deploy_id : %s' % self.deploy_id

# Client and database attributes dictionary
clinetHead = {u'deployId', u'description', u'deployedAt', u'createdAt', u'updatedAt'}
tableChangeDic = {
    "deployId":"deploy_id",
    "description":"description",
    "deployedAt":"deployed_at",
    "createdAt":"created_at",
    "updatedAt":"updated_at"
}

intList = {u'deployId'}

# db.create_all()


from django import forms
from django.core.validators import *
from .models import *

QuestionChoice = (
    ('Multiple Choice','Multiple Choice'), ('Boolean Type','Boolean Type'), ('Rating Scale','Rating Scale'),
    ('Text/Paragraph','Text/Paragraph'), ('Number Input','Number Input'), ('Picture Input','Picture Input'),
)

SourceChoice = (
    ('1','Manually'),('2','Import CSV file'),
)

class TeacherForm(forms.ModelForm):
    Name = forms.CharField(label='Teacher Name')
    Description = forms.CharField(label='Task Description')
    NumberOfQuestions = forms.IntegerField(label='No. of questions')
    Source = forms.ChoiceField(label="How would you like provide data ",widget=forms.RadioSelect,choices=SourceChoice)
    File = forms.FileField(label="Choose file to upload(*Not Requied in manual mode)",required=False)
    class Meta:
        model= Project
        fields= '__all__'

class TaskForm(forms.ModelForm):
    Question = forms.CharField(label='Question',max_length=500,required=True)
    QuestionType = forms.ChoiceField(label='Question Type',choices=QuestionChoice)
    AnswerOptions = forms.CharField(label='Answer Option',)
    class Meta:
        model = Task
        fields= ['Question','QuestionType','AnswerOptions']

class StudentMultipleChoiceForm(forms.ModelForm):
    def __init__(self, user, *args, **kwargs):
        super(StudentMultipleChoiceForm, self).__init__(*args, **kwargs)
        teacher = Temp.objects.latest('TeacherName')
        question_list=list(Temp.objects.filter(TeacherName=teacher.TeacherName))
        choice=question_list[0].AnswerOptions.split(',')
        MultipleChoice =(
            (choice[0],choice[0]),(choice[1],choice[1]),(choice[2],choice[2]),(choice[3],choice[3]),
        )
        self.fields['Answers'] = forms.ChoiceField(label="Answer",widget=forms.RadioSelect,choices= MultipleChoice)
    class Meta:
        model=Answer
        fields=['Answers']

class StudentBooleanChoiceForm(forms.ModelForm):
    def __init__(self, user,*args, **kwargs):
        super(StudentBooleanChoiceForm, self).__init__(*args, **kwargs)
        teacher = Temp.objects.latest('TeacherName')
        question_list=list(Temp.objects.filter(TeacherName=teacher.TeacherName))
        choice=question_list[0].AnswerOptions.split(',')
        MultipleChoice =(
            (choice[0],choice[0]),(choice[1],choice[1]),
        )
        self.fields['Answers'] = forms.ChoiceField(label="Answer",widget=forms.RadioSelect,choices= MultipleChoice)
    class Meta:
        model=Answer
        fields=['Answers']

class StudentRatingScaleForm(forms.ModelForm):
    def __init__(self, user, *args, **kwargs):
        super(StudentRatingScaleForm, self).__init__(*args, **kwargs)
        teacher = Temp.objects.latest('TeacherName')
        question_list=list(Temp.objects.filter(TeacherName=teacher.TeacherName))
        ratings=question_list[0].AnswerOptions.split(',')
        RatingChoice = []
        for i in range(0,len(ratings)):
            RatingChoice.append((ratings[i],ratings[i]))
        self.fields['Answers'] = forms.ChoiceField(label="Answer",widget=forms.RadioSelect,choices= RatingChoice)
    class Meta:
        model=Answer
        fields=['Answers']

class StudentParagraphForm(forms.ModelForm):
    def __init__(self,user, *args, **kwargs):
        super(StudentParagraphForm, self).__init__(*args, **kwargs)
        teacher = Temp.objects.latest('TeacherName')
        question_list=list(Temp.objects.filter(TeacherName=teacher.TeacherName))
        max_characters=question_list[0].AnswerOptions
        self.fields['Answers'] = forms.CharField(label="Answer",widget=forms.Textarea,max_length=int(max_characters))
    class Meta:
        model=Answer
        fields=['Answers']

class StudentNumberInputForm(forms.ModelForm):
    def __init__(self, user, *args, **kwargs):
        super(StudentNumberInputForm, self).__init__(*args, **kwargs)
        teacher = Temp.objects.latest('TeacherName')
        question_list=list(Temp.objects.filter(TeacherName=teacher.TeacherName))
        number_range=question_list[0].AnswerOptions.split(',')
        self.fields['Answers'] = forms.IntegerField(label="Answer",validators=[MaxValueValidator(int(number_range[1])),
            MinValueValidator((int(number_range[0])))])
    class Meta:
        model=Answer
        fields=['Answers']

# class StudentPictureInputForm(forms.ModelForm):
#     def __init__(self, user,question_list, *args, **kwargs):
#         super(StudentPictureInputForm, self).__init__(*args, **kwargs)
#
#         self.fields['Choice'] = forms.
#     class Meta:
#         model=Answer
#         fields=[]


#! /volume1/@appstore/python3/bin/python3
#coding=utf-8
#%%
import numpy as np
import pandas as pd
from pyecharts.charts import Kline, Line, Bar, EffectScatter, Boxplot, Grid
from pyecharts import options as opts
from pyecharts.globals import SymbolType, CurrentConfig
from datetime import datetime, timedelta
from pytz import timezone
import sys
from DataAgent import DataAgent
from DBObj import Stock_poll, Index_poll
from Radar import Radar
import libs
import config as cfg
from itertools import product

class Analytics:
    def __init__ (self,daObj):
        self.da = daObj
        self.poll_aggHistRec = pd.DataFrame() # sec code 级别的聚合数据，集合包含证券及指数
        self.poll_aggStkPoll = pd.DataFrame() # poll time index 级别的聚合数据，只包含股票
        self.poll_index = pd.DataFrame()
        self.poll_stock = pd.DataFrame()
        self.pollchart = None
        self.kchart = None
        self.macrochart = None
        self.kchartdf = pd.DataFrame() # k线图原始数据

    def candlechart(self,str_tscode=None,subset='masterCandle',dat_cursor=None,MA=[5,10,21], renderfile=False) -> Grid:
        str_tscode = DataAgent.formatCode(str_tscode)
        int_chartsize = 500
        if str_tscode == False or str_tscode is None:
            print('invalid sec code provided...')
            return False
        dat_cursor = datetime.now().astimezone(timezone(cfg.STR_TIMEZONE)) if dat_cursor is None else dat_cursor
        closestTradeDate = self.da.int_findClosestTradeDate(datetime.strftime(dat_cursor,'%Y%m%d') )
        tpl_dateindex = tuple(str(i) for i in self.da.df_calTbl.loc[self.da.df_calTbl['cal_date'] <= closestTradeDate]['cal_date'][-int_chartsize:] )

        # 连接储存对象
        if subset=='masterCandle':
            data_store = libs.tryOpenH5(cfg.H5_FILE_PRE,mode='r')
        else:
            data_store = libs.tryOpenH5('{}{}.dat'.format(cfg.H5_FILE_PATH,subset),mode='r')

        # 初始化交易日表格，作为标准交易日序列join个股交易数据
        df_toPlot = data_store['masterCandle'].loc[data_store['masterCandle']['ts_code']==str_tscode]
        df_dateindex = pd.DataFrame({'trade_caldate': tpl_dateindex}).astype(int)
        data_store.close()
        try:
            str_secname = self.da.df_lookupTbl.loc[self.da.df_lookupTbl['ts_code']==str_tscode]['name'].values[0]
            str_sectype = self.da.df_lookupTbl.loc[self.da.df_lookupTbl['ts_code']==str_tscode]['sec_type'].values[0]
        except Exception as e:
            return libs.log_csv(str_cat='WARNING',str_op='candlechart',str_desc='security code {} does not exist in basic info table...'.format(str_tscode))
        str_plotname = '{} - {}'.format(str_tscode,str_secname)

        # 交易日时间轴标准化，补全停牌日,当日live数据等k线
        '''------ 获取实时交易数据部分--------------------'''
        int_liveDate = self.da.int_findClosestTradeDate(datetime.strftime(datetime.now().astimezone(timezone(cfg.STR_TIMEZONE)),'%Y%m%d') )
        if int_liveDate<=closestTradeDate and int_liveDate>df_toPlot['trade_date'].max():
            df_liveQuote = self.da.query('quote_now',[self.da.formatCode(str_tscode,False),])
            if len(df_liveQuote)>0:
                df_liveQuote.rename(columns={'open':'adj_open',
                                'high':'adj_high',
                                'low':'adj_low',
                                'price':'adj_close',
                                'volume':'vol',
                                'amount':'amount'}, inplace=True)
                df_liveQuote=df_liveQuote.astype(dtype= {'adj_open':'float','adj_high':'float','adj_low':'float','adj_close':'float','vol':'float','amount':'float'})
                df_liveQuote['trade_date'] = int_liveDate
                df_liveQuote['ts_code'] = str_tscode
                df_liveQuote['vol'] = df_liveQuote['vol']/100 if str_sectype=='stk' or str_tscode[:3]=='399' else  df_liveQuote['vol']
                df_liveQuote['amount'] = df_liveQuote['amount']/1000
            df_toPlot = df_toPlot.append(df_liveQuote[['ts_code','trade_date','adj_open','adj_high','adj_low','adj_close','vol','amount']],sort=False)
        '''------ 结束 获取实时交易数据部分--------------------'''

        df_toPlot = pd.merge(df_dateindex, df_toPlot, left_on='trade_caldate', right_on='trade_date', how='left')
        df_toPlot['close'].fillna(method='ffill',inplace=True)
        df_toPlot['ts_code'].fillna(method='ffill',inplace=True)
        df_toPlot['klineSML'].fillna('NNNN',inplace=True)
        df_toPlot['adj_close'].fillna(method='ffill',inplace=True)
        df_toPlot['vol'].fillna(value=0,inplace=True)
        df_toPlot['amount'].fillna(value=0,inplace=True)
        df_toPlot['open'].fillna(df_toPlot['close'],inplace=True)
        df_toPlot['high'].fillna(df_toPlot['close'],inplace=True)
        df_toPlot['low'].fillna(df_toPlot['close'],inplace=True)
        df_toPlot['adj_open'].fillna(df_toPlot['adj_close'],inplace=True)
        df_toPlot['adj_high'].fillna(df_toPlot['adj_close'],inplace=True)
        df_toPlot['adj_low'].fillna(df_toPlot['adj_close'],inplace=True)
        self.kchartdf = df_toPlot #输出至可访问对象属性中

        '''-----------------画图部分---------------------'''
        lst_ohlcv=df_toPlot.loc[:,['adj_open','adj_close','adj_low','adj_high']].values.tolist()
        lst_vol=list(df_toPlot['vol'].values)
        lst_amount=list(df_toPlot['amount'].values)
        lst_peaks=list(abs(df_toPlot['peaks'].values))
        # lst_pivotdown=[float('nan') if i<0 else i for i in list(df_toPlot['pivots'].values)]
        lst_pivotdown=[float('nan') if i<0 else i for i in list(df_toPlot['valid_pivots'].values)]
        # lst_pivotup=[float('nan') if i>0 else abs(i) for i in list(df_toPlot['pivots'].values)]
        lst_pivotup=[float('nan') if i>0 else abs(i) for i in list(df_toPlot['valid_pivots'].values)]
        lst_xaxis=list(df_toPlot['trade_caldate'].astype(str))

        def calculate_ma(day_count: int, d):
            result: List[Union[float, str]] = []
            for i in range(len(d)):
                if i < day_count:
                    result.append("-")
                    continue
                sum_total = 0.0
                for j in range(day_count):
                    sum_total += float(d[i - j][1])
                result.append(abs(float("%.3f" % (sum_total / day_count))))
            return result

        kline = (Kline()
            .add_xaxis(lst_xaxis)
            .add_yaxis(series_name=str_secname, y_axis=lst_ohlcv,
                        markpoint_opts=opts.MarkPointOpts(
                            data=[opts.MarkPointItem(type_="min",value_dim="close"),
                                opts.MarkPointItem(type_="max",value_dim="close")],
                            symbol_size = [20, 20], #表示标记宽为 20，高为 10
                        ),
                        itemstyle_opts=opts.ItemStyleOpts(
                            color="#ec0000",
                            color0="#00da3c",
                            border_color="#ec0000",
                            border_color0="#00da3c",
                        ),)
            .set_global_opts(
                title_opts=opts.TitleOpts(
                    title=str_plotname,
                    subtitle='MA='+str(MA),
                ),
                xaxis_opts=opts.AxisOpts(type_="category"),
                yaxis_opts=opts.AxisOpts(
                    is_scale=True,
                    splitarea_opts=opts.SplitAreaOpts(
                        is_show=True, areastyle_opts=opts.AreaStyleOpts(opacity=1)
                    ),
                ),
                legend_opts=opts.LegendOpts(is_show=True, pos_top='10%', pos_left="center"),
                datazoom_opts=[
                    opts.DataZoomOpts(
                        is_show=False,
                        type_="inside",
                        xaxis_index=[0,1,2],
                        range_start=75,
                        range_end=100,
                    ),
                    opts.DataZoomOpts(
                        is_show=True,
                        xaxis_index=[0,1,2],
                        type_="slider",
                        pos_top="90%",
                        range_start=75,
                        range_end=100,
                    ),
                ],
                tooltip_opts=opts.TooltipOpts(
                    trigger="axis",
                    axis_pointer_type="cross",
                    background_color="rgba(245, 245, 245, 0.8)",
                    border_width=1,
                    border_color="#ccc",
                    textstyle_opts=opts.TextStyleOpts(color="#000",font_size=10),
                ),
                # 阴量绿阳量红
                visualmap_opts=opts.VisualMapOpts(
                    is_show=False,
                    dimension=2,
                    series_index=list(map(lambda x: x+len(MA),[4,5])), #动态算出vol和amt柱状图的series index
                    is_piecewise=True,
                    pieces=[
                        {"value": 1, "color": "#ec0000"},
                        {"value": -1, "color": "#00da3c"},
                    ],
                ),
                axispointer_opts=opts.AxisPointerOpts(
                    is_show=True,
                    link=[{"xAxisIndex": "all"}],
                    label=opts.LabelOpts(background_color="#777"),
                ),
                brush_opts=opts.BrushOpts(
                    x_axis_index="all",
                    brush_link="all",
                    out_of_brush={"colorAlpha": 0.1},
                    brush_type="lineX",
                ),
            )
        )
        trendline = (
            Line()
            .add_xaxis(lst_xaxis)
            .add_yaxis('高低点', lst_peaks,
                        itemstyle_opts=opts.ItemStyleOpts(color="green"))
        )
        for i in MA:
            if i is not None:
                trendline.add_yaxis(
                        series_name='MA'+str(i),
                        y_axis=calculate_ma(day_count=i, d=lst_ohlcv),
                        is_smooth=True,
                        is_hover_animation=False,
                        linestyle_opts=opts.LineStyleOpts(width=1, opacity=0.5),
                        label_opts=opts.LabelOpts(is_show=False),
                )
        trendline.set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        trendline.set_global_opts(legend_opts=opts.LegendOpts(is_show=False))

        keyPoints = (
            EffectScatter()
            .add_xaxis(lst_xaxis)
            .add_yaxis("末跌高", lst_pivotdown,symbol=SymbolType.ARROW,symbol_rotate=180,symbol_size=5,itemstyle_opts=opts.ItemStyleOpts(color="purple"))
            .add_yaxis("末升低", lst_pivotup,symbol=SymbolType.ARROW,symbol_size=5,itemstyle_opts=opts.ItemStyleOpts(color="blue"))
            .set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        )
        vol_bar = (
            Bar()
            .add_xaxis(lst_xaxis)
            .add_yaxis(
                series_name="交易量",
                yaxis_data=[
                [i, lst_vol[i], 1 if lst_ohlcv[i][0] < lst_ohlcv[i][1] else -1]
                for i in range(len(lst_vol))
                ],
                xaxis_index=1,
                yaxis_index=1,
                label_opts=opts.LabelOpts(is_show=False),
            )
            .set_global_opts(
                xaxis_opts=opts.AxisOpts(
                    type_="category",
                    is_scale=True,
                    grid_index=1,
                    boundary_gap=False,
                    axisline_opts=opts.AxisLineOpts(is_on_zero=False),
                    axistick_opts=opts.AxisTickOpts(is_show=False),
                    splitline_opts=opts.SplitLineOpts(is_show=False),
                    axislabel_opts=opts.LabelOpts(is_show=False),
                    split_number=20,
                    min_="最低",
                    max_="最高",
                ),
                yaxis_opts=opts.AxisOpts(
                    grid_index=1,
                    is_scale=True,
                    split_number=2,
                    axislabel_opts=opts.LabelOpts(is_show=False),
                    axisline_opts=opts.AxisLineOpts(is_show=False),
                    axistick_opts=opts.AxisTickOpts(is_show=False),
                    splitline_opts=opts.SplitLineOpts(is_show=False),
                ),
                legend_opts=opts.LegendOpts(is_show=False),
            )
            # .add_yaxis("交易量", lst_vol,itemstyle_opts=opts.ItemStyleOpts(color="#456A76"))
        )
        amnt_bar = (
            Bar()
            .add_xaxis(lst_xaxis)
            .add_yaxis(
                series_name="交易额",
                yaxis_data=[
                [i, lst_amount[i], 1 if lst_ohlcv[i][0] < lst_ohlcv[i][1] else -1]
                for i in range(len(lst_amount))
                ],
                xaxis_index=2,
                yaxis_index=2,
                label_opts=opts.LabelOpts(is_show=False),
            )
            .set_global_opts(
                xaxis_opts=opts.AxisOpts(
                    type_="category",
                    is_scale=True,
                    grid_index=2,
                    boundary_gap=False,
                    axisline_opts=opts.AxisLineOpts(is_on_zero=False),
                    axistick_opts=opts.AxisTickOpts(is_show=False),
                    splitline_opts=opts.SplitLineOpts(is_show=False),
                    axislabel_opts=opts.LabelOpts(is_show=False),
                    split_number=20,
                    min_="最低",
                    max_="最高",
                ),
                yaxis_opts=opts.AxisOpts(
                    grid_index=2,
                    is_scale=True,
                    split_number=2,
                    axislabel_opts=opts.LabelOpts(is_show=False),
                    axisline_opts=opts.AxisLineOpts(is_show=False),
                    axistick_opts=opts.AxisTickOpts(is_show=False),
                    splitline_opts=opts.SplitLineOpts(is_show=False),
                ),
                legend_opts=opts.LegendOpts(is_show=False),
            )
            # .add_yaxis("交易额", lst_amount,itemstyle_opts=opts.ItemStyleOpts(color="#456A76"))
        )
        priceChart = kline.overlap(trendline).overlap(keyPoints)
        gridChart = Grid()
        gridChart.add(
            priceChart,
            grid_opts=opts.GridOpts(pos_left="10%", pos_right="8%", pos_bottom='40%'),
        )
        gridChart.add(
            vol_bar,
            grid_opts=opts.GridOpts(pos_left="10%", pos_right="8%", pos_top="60%", height='15%'),
        )
        gridChart.add(
            amnt_bar,
            grid_opts=opts.GridOpts(pos_left="10%", pos_right="8%", pos_top="75%"),
        )

        fname = '{}{}.html'.format(cfg.PATH_ANAFILE,'kline')
        gridChart.render(fname) if renderfile else None
        self.kchart = gridChart # 将结果输出到analytics对象属性中用于function以外的调用
        return self

    def pollsummary(self,subset='masterCandle',dat_cursor=None,renderfile=False):
        # 初始化交易时间x轴
        heartbeat = 300
        # 如果未提供复盘时间游标，则默认使用最近交易日至收盘收盘时间的数据
        if dat_cursor is None:
            dat_cursor = datetime.now().astimezone(timezone(cfg.STR_TIMEZONE))
            int_latestViewDate = self.da.int_findClosestTradeDate(datetime.strftime(dat_cursor,'%Y%m%d') )
            int_cursorDateTime = int(str(int_latestViewDate)+'150100')
            int_cursorDateStart = int(str(int_latestViewDate)+'000000')
        # 如提供复盘游标时间，则该时间为游标卡尺的最后时间，如日期为非交易日则自动调整为最近交易日的收盘时间
        else:
            int_cursorDate = datetime.strftime(dat_cursor,'%Y%m%d')
            int_latestViewDate = self.da.int_findClosestTradeDate(int_cursorDate)
            if int_cursorDate==int_latestViewDate: # 如相等说明提供日期为有效交易日，则使用提供的小时分钟秒为游标终点时间，否则将自动替换为最近交易日的收盘时间点
                int_cursorDateTime = int(datetime.strftime(dat_cursor,'%Y%m%d%H%M%S'))
                int_cursorDateStart = int(datetime.strftime(dat_cursor,'%Y%m%d')+'000000')
            else:
                int_cursorDateTime = int(str(int_latestViewDate) +'150100')
                int_cursorDateStart = int(str(int_latestViewDate) +'000000')
        beatRange = Radar.pop_pulseRange(int_latestViewDate,heartbeat)
        df_stockPoll = pd.read_sql(DataAgent.dbsession.query(Stock_poll).filter(Stock_poll.time_index>=int_cursorDateStart,
                                        Stock_poll.time_index<=int_cursorDateTime,
                                        Stock_poll.volume!=0).statement,
                                    DataAgent.dbsession.bind)
        df_indexPoll = pd.read_sql(DataAgent.dbsession.query(Index_poll).filter(Index_poll.time_index>=int_cursorDateStart,
                                        Index_poll.time_index<=int_cursorDateTime,
                                        Index_poll.volume!=0).statement,
                                    DataAgent.dbsession.bind)
        '''
        data_store = libs.tryOpenH5(cfg.H5_FILE_POLL,mode='r')
        if 'stockPoll' in data_store:
            df_stockPoll = data_store['stockPoll']
            df_stockPoll = df_stockPoll[df_stockPoll['volume']!=0] # 过滤掉数据中非交易中的证券
            df_stockPoll = df_stockPoll[(df_stockPoll['time_index']<=int_cursorDateTime)&(df_stockPoll['time_index']>=int_cursorDateStart)] # 过滤掉和画图期间无关的数据
        else:
            df_stockPoll = pd.DataFrame()
        df_indexPoll = data_store['indexPoll'] if 'indexPoll' in data_store else pd.DataFrame()
        data_store.close()
        '''

        data_store = libs.tryOpenH5(cfg.H5_FILE_PRE,mode='r')
        # 如果按照输入日期没有找到当日任何盘中数据，则从入库历史数据中找到当日收盘数据用于填充
        if len(df_indexPoll)==0:
            df_indexPoll = data_store['masterCandle'][data_store['masterCandle']['trade_date']==int_latestViewDate]
            df_indexPoll['time_index'] = int_cursorDateTime
            df_indexPoll['pct_change'] = round(df_indexPoll['adj_close']/df_indexPoll['pre_close']*100,2)
            df_indexPoll['per'] = np.nan
        if len(df_stockPoll)==0:
            df_stockPoll = data_store['masterCandle'][data_store['masterCandle']['trade_date']==int_latestViewDate]
            df_stockPoll['time_index'] = int_cursorDateTime
            df_stockPoll['pct_change'] = round((df_stockPoll['adj_close']-df_stockPoll['pre_close'])/df_stockPoll['pre_close']*100,2)
            df_stockPoll.rename(columns={'pe': 'per'}, inplace=True)
        # 如果入库历史数据中仍未找到任何数据则退出分析程序
        if any([len(df_stockPoll)==0,len(df_indexPoll)==0]):
                data_store.close()
                print('stockPoll or indexPoll does not exist for the cursor date in the offline data store, process exiting...')
                return False

        ''' ------读取security code级别的聚合数据------- '''
        if subset=='masterCandle':
            pass #data_store = libs.tryOpenH5(cfg.H5_FILE_PRE,mode='r')
        else:
            data_store.close() # 关闭之前打开的master data file
            data_store = libs.tryOpenH5('{}{}.dat'.format(cfg.H5_FILE_PATH,subset),mode='r')
        df_histRec = data_store['AggByCode']
        data_store.close()
        self.poll_aggHistRec = df_histRec # sec code 级别的聚合数据，集合包含证券及指数,输出至对象变量
        ''' ------读取security code级别的聚合结束------- '''

        sr_intBeatIndex = pd.Series(beatRange.strftime('%Y%m%d%H%M%S').astype('int64')).rename(index='time_index')
        df_index = pd.merge(sr_intBeatIndex,df_indexPoll,how='left',left_on='time_index',right_on='time_index',suffixes=['','1'])

        df_stock = pd.merge(sr_intBeatIndex,df_stockPoll,how='left',left_on='time_index',right_on='time_index',suffixes=['','1'])
        df_stock = pd.merge(df_stock,df_histRec,how='left',left_on='ts_code',right_index=True,suffixes=['','1'])

        self.poll_index = df_index #将结果输出至analyitics对象用于方法外的访问
        self.poll_stock = df_stock #将结果输出至analyitics对象用于方法外的访问
        # libs.df_csv(cfg.PATH_BUGFILE,(df_stock,))
        ''' --------所有by time index级别的聚合在这里完成 -----'''
        def popPctList(x):
            # 移除涨跌幅超过+- 10%的, 新股不受10%涨跌幅限制因此会干扰box chart显示
            return [0]*3 if len(x)<3 else [i for i in x if abs(i)<11]
        def bigBox(x, up=True):
            # 中阳（阴）以上k线数量
            if up:
                return list(x.loc[(x['box_size']>0) & (x['klineSML'].str.match(DataAgent.re_bigUpBoxPat))]['ts_code'])
            else:
                return list(x.loc[(x['box_size']<0) & (x['klineSML'].str.match(DataAgent.re_bigDnBoxPat))]['ts_code'])
        def breakThrough(x,up=True):
            if up: # 找到上穿有效末跌高点的
                return list(x.loc[x['close']>=x['validPiv_dnHigh']]['ts_code'])
            else: # 找到跌破有效末升低点的
                return list(x.loc[x['close']<=x['validPiv_upLow']]['ts_code'])
        df_stockBeatGrp = df_stock.groupby('time_index')
        df_stockBeatRollup = df_stockBeatGrp.agg({'ts_code': ['count'],
                                'pct_change': [popPctList],
                                'per': ['median']}) # function放入中括号内会生成两个层级列名，方便后面join成新列名.如无中括号将只产生一层列名
        df_stockBeatRollup.columns = ['_'.join(x) for x in df_stockBeatRollup.columns.ravel()]
        df_stockBeatRollup['upThruList'] = df_stockBeatGrp.apply(lambda x: breakThrough(x,up=True)) if len(df_histRec) > 0 else np.nan
        df_stockBeatRollup['dnThruList']= df_stockBeatGrp.apply(lambda x: breakThrough(x,up=False)) if len(df_histRec) > 0 else np.nan
        df_stockBeatRollup['upThruCount'] = df_stockBeatRollup.apply(lambda row: len(row['upThruList']),axis=1) if len(df_histRec) > 0 else np.nan
        df_stockBeatRollup['dnThruCount']= df_stockBeatRollup.apply(lambda row: len(row['dnThruList']),axis=1) if len(df_histRec) > 0 else np.nan

        df_stockBeatRollup['upBigBoxList'] = df_stockBeatGrp.apply(lambda x: bigBox(x,up=True))
        df_stockBeatRollup['dnBigBoxList'] = df_stockBeatGrp.apply(lambda x: bigBox(x,up=False))
        df_stockBeatRollup['upBigBoxCount'] = df_stockBeatRollup.apply(lambda row: len(row['upBigBoxList']),axis=1) if len(df_histRec) > 0 else np.nan
        df_stockBeatRollup['dnBigBoxCount'] = df_stockBeatRollup.apply(lambda row: len(row['dnBigBoxList']),axis=1) if len(df_histRec) > 0 else np.nan
        ''' --------by time index级别的聚合结束 -------------------------------'''

        df_stockBeatRollup.astype(dtype= {'ts_code_count':'int32','upThruCount':'int32','dnThruCount':'int32','upBigBoxCount':'int32','dnBigBoxCount':'int32'})
        self.poll_aggStkPoll = df_stockBeatRollup.loc[df_stockBeatRollup['ts_code_count']>0] # 只将有效结果输出到analytics对象属性中用于function以外的调用

        ''' ------------------作图区域---------------------------'''
        per_median = round(self.poll_aggStkPoll.loc[self.poll_aggStkPoll.index==self.poll_aggStkPoll.index.max()].per_median.values[0],2)

        lst_xaxis = ['{}:{}:{}'.format(i.hour,i.minute,i.second) for i in beatRange]
        marketSummaryBoxPlot = Boxplot()
        lst_yaxis = np.around(marketSummaryBoxPlot.prepare_data(df_stockBeatRollup['pct_change_popPctList'].values), decimals=2).tolist()

        marketSummaryBoxPlot.add_xaxis(lst_xaxis)
        marketSummaryBoxPlot.add_yaxis("%", lst_yaxis)
        marketSummaryBoxPlot.set_global_opts(title_opts=opts.TitleOpts(title='整体涨跌幅分布 - {} (最后市盈率中位数:{})'.format(int_latestViewDate,str(per_median))),
            yaxis_opts=opts.AxisOpts(name="涨幅",min_=-10,max_=10),
            xaxis_opts=opts.AxisOpts(name="",
                axislabel_opts=opts.LabelOpts(is_show=False),
                axisline_opts=opts.AxisLineOpts(is_show=True),
                axistick_opts=opts.AxisTickOpts(is_show=True),
                splitline_opts=opts.SplitLineOpts(is_show=False),),
            legend_opts=opts.LegendOpts(is_show=False),
        )

        lst_upThruBar = df_stockBeatRollup['upThruCount'].values.tolist()
        lst_dnThruBar = df_stockBeatRollup['dnThruCount'].values.tolist()
        lst_bigUpBoxBar = df_stockBeatRollup['upBigBoxCount'].values.tolist()
        lst_bigDnBoxBar = df_stockBeatRollup['dnBigBoxCount'].values.tolist()

        pivotCountChart = (
            Bar()
            .add_xaxis(lst_xaxis)
            .add_yaxis(
                series_name="向上突破数量",
                yaxis_data=lst_upThruBar,
                label_opts=opts.LabelOpts(is_show=False),
                itemstyle_opts=opts.ItemStyleOpts(color="#FD625E"),
            )
            .add_yaxis(
                series_name="向下突破数量",
                xaxis_index=1,
                yaxis_index=1,
                yaxis_data=lst_dnThruBar,
                label_opts=opts.LabelOpts(is_show=False),
                itemstyle_opts=opts.ItemStyleOpts(color="#01B8AA"),
            )
            .add_yaxis(
                series_name="长阳线数量",
                xaxis_index=1,
                yaxis_index=1,
                yaxis_data=lst_bigUpBoxBar,
                label_opts=opts.LabelOpts(is_show=False),
                itemstyle_opts=opts.ItemStyleOpts(color="red"),
            )
            .add_yaxis(
                series_name="长阴线数量",
                xaxis_index=1,
                yaxis_index=1,
                yaxis_data=lst_bigDnBoxBar,
                label_opts=opts.LabelOpts(is_show=False),
                itemstyle_opts=opts.ItemStyleOpts(color="green"),
            )
            .set_global_opts(
                xaxis_opts=opts.AxisOpts(name="小时/分钟",),
                yaxis_opts=opts.AxisOpts(
                    axislabel_opts=opts.LabelOpts(is_show=True),
                    axisline_opts=opts.AxisLineOpts(is_show=True),
                    axistick_opts=opts.AxisTickOpts(is_show=True),
                    splitline_opts=opts.SplitLineOpts(is_show=True),
                ),
                legend_opts=opts.LegendOpts(is_show=True,pos_bottom='0%', pos_left="center"),
            )
        )
        gridChart = Grid()
        gridChart.add(
            marketSummaryBoxPlot,
            grid_opts=opts.GridOpts(pos_left="10%", pos_right="8%", pos_bottom='45%'),
        )
        gridChart.add(
            pivotCountChart,
            grid_opts=opts.GridOpts(pos_left="10%", pos_right="8%", pos_top="55%"),
        )
        fname = '{}marketSummary{}.html'.format(cfg.PATH_ANAFILE,int_latestViewDate)
        gridChart.render(fname) if renderfile else None
        return gridChart

    def corelationMacro(self,dat_cursor=None,lst_tscode=['399002.SZ'],lst_macro=['shibor.on','PIMon.cpi'],lst_findmedian=['pe',()],renderfile=False) -> Line:
        lst_findmedian[1] = tuple(self.da.df_stockTbl['ts_code']) if lst_findmedian[1] is None else lst_findmedian[1]

        if dat_cursor is None:
            dat_cursor = datetime.now().astimezone(timezone(cfg.STR_TIMEZONE))
        int_latestViewDate = self.da.int_findClosestTradeDate(datetime.strftime(dat_cursor,'%Y%m%d') )
        df_baseCal = self.da.df_calTbl.loc[self.da.df_calTbl['cal_date']<=int_latestViewDate]['cal_date'].copy()
        lst_xaxis = ['{}/{}/{}'.format(i[:4],i[4:6],i[6:]) for i in df_baseCal.values.astype('str')]

        linechart = (
                Line()
                .add_xaxis(lst_xaxis)
        )

        for code in lst_tscode:
            str_tscode = DataAgent.formatCode(code)
            df_target = self.da.query('load_alldaily',tpl_dateindex=tuple(df_baseCal),ts_code=str_tscode)
            if len(df_target)>0:
                df_toPlot = pd.merge(left=df_baseCal, right=df_target, how='left', left_on='cal_date', right_on='trade_date')
            else:
                continue
            linename = '{} - {}'.format(str_tscode,self.da.df_lookupTbl.loc[self.da.df_lookupTbl['ts_code']==str_tscode]['name'].values[0])
            lst_values = list(df_toPlot['close'])
            lst_values = list(Analytics.normByRange(lst_values))

            linechart.add_yaxis(linename, lst_values,
                is_smooth=True,
                is_hover_animation=False,
                linestyle_opts=opts.LineStyleOpts(width=3, opacity=0.3),
                label_opts=opts.LabelOpts(is_show=False),
                itemstyle_opts=opts.ItemStyleOpts(
                        color="#00da3c",
                ),
            )
            try:
                lst_pe = df_toPlot['pe'].values.tolist()
                lst_pb = df_toPlot['pb'].values.tolist()
                lst_pepb = [round(lst_pe[index]/value,2) for index, value in enumerate(lst_pb) if value!=0]
                linechart.add_yaxis(str_tscode+"_PE", lst_pe, yaxis_index=1, #倍数使用右y轴
                    is_smooth=True,
                    is_hover_animation=False,
                    linestyle_opts=opts.LineStyleOpts(width=1),
                    label_opts=opts.LabelOpts(is_show=False),
                )
                linechart.add_yaxis(str_tscode+"_PE/PB", lst_pepb, yaxis_index=1, #倍数使用右y轴
                    is_smooth=True,
                    is_hover_animation=False,
                    linestyle_opts=opts.LineStyleOpts(width=1),
                    label_opts=opts.LabelOpts(is_show=False),
                )
            except:
                continue

        df_shibor = self.da.query('load_shibor')
        # print('Supported shibor columns: {}'.format(df_shibor.columns))
        df_pimon = self.da.query('load_piMon')
        # print('Supported macro eco columns: {}'.format(df_pimon.columns))

        for item in lst_macro:
            try:
                topic,colname = item.split('.')
            except:
                print('WARNING! input code {} not understood...'.format(item))
            if topic=='shibor':
                df_target = df_shibor
                maxarr = df_target.max(axis=0, skipna=True).values[1:]
                minarr = df_target.min(axis=0, skipna=True).values[1:]
                deltaRng = max(maxarr)-min(minarr) # overwrite the normalization function delta to put all shibor rates comparable
            elif topic=='pimon':
                df_target = df_pimon
                deltaRng = None
            else:
                print('WARNING! subject code {} is not understood...'.format(topic))
            if len(df_target)>0:
                df_toPlot = pd.merge(df_baseCal, df_target, how='left', left_on='cal_date', right_on='date')
                df_toPlot[colname] = df_toPlot[colname].bfill()
            else:
                continue
            linename = item
            lst_values = list(df_toPlot[colname])
            lst_values = list(Analytics.normByRange(lst_values,delta=deltaRng))
            linechart.add_yaxis(linename, lst_values, # 所有等比例缩放的折线都放在linechart里，使用左侧y轴
                is_smooth=True,
                is_hover_animation=False,
                linestyle_opts=opts.LineStyleOpts(width=1,opacity=0.5),
                label_opts=opts.LabelOpts(is_show=False),
            )
        str_cat = lst_findmedian[0]
        df_result = pd.DataFrame()
        for str_tscode in lst_findmedian[1]:
            try:
                df = self.da.query('load_alldaily',tpl_dateindex=tuple(df_baseCal),ts_code=str_tscode)[['ts_code','trade_date',str_cat]]
                print('{} csv file loaded for {} trend analysis...'.format(str_tscode,str_cat),end='\r')
            except Exception as e:
                print('WARNING! {} csv file not found...'.format(str_tscode))
                continue
            df_result = pd.concat([df_result,df])
        df_result.dropna(subset=[str_cat],inplace=True)
        df_result = df_result.loc[df_result['trade_date']>19940301]
        # libs.df_csv(cfg.PATH_BUGFILE,(df_result.groupby('trade_date')['ts_code'].count(),))
        # print(df_result.groupby('trade_date')['ts_code'].count())
        df_target = df_result.groupby('trade_date')[str_cat].median()
        if len(df_target)>0:
            df_toPlot = pd.merge(left=df_baseCal, right=df_target, how='left', left_on='cal_date', right_on='trade_date')
            linename = '{}-median'.format(str_cat)
            lst_values = list(df_toPlot[str_cat])
            lst_values = list(lst_values)
            linechart.add_yaxis(linename, lst_values, yaxis_index=1,
                is_smooth=True,
                is_hover_animation=False,
                linestyle_opts=opts.LineStyleOpts(width=1),
                label_opts=opts.LabelOpts(is_show=False),
            )
            linechart.extend_axis( # 所有保持原比例的折线都放在linechart2里，使用右侧y轴
                yaxis=opts.AxisOpts(
                    name="倍数",
                    type_="value",
                    position="right",
                    axisline_opts=opts.AxisLineOpts(
                        linestyle_opts=opts.LineStyleOpts(color="black")
                    ),
                    axislabel_opts=opts.LabelOpts(formatter="{value}倍"),
                )
            )
            linechart.set_global_opts(
                title_opts=opts.TitleOpts(
                    title='宏观比较',
                ),
                xaxis_opts=opts.AxisOpts(type_="category"),
                yaxis_opts=opts.AxisOpts(name="等比例",
                    is_scale=True,
                ),
                legend_opts=opts.LegendOpts(is_show=True, pos_top='5%', pos_left="center"),
                datazoom_opts=[
                    opts.DataZoomOpts(
                        is_show=False,
                        type_="inside",
                        xaxis_index=[0],
                        range_start=75,
                        range_end=100,
                    ),
                    opts.DataZoomOpts(
                        is_show=True,
                        xaxis_index=[0],
                        type_="slider",
                        pos_top="90%",
                        range_start=75,
                        range_end=100,
                    ),
                ],
                tooltip_opts=opts.TooltipOpts(
                    trigger="axis",
                    axis_pointer_type="cross",
                    background_color="rgba(245, 245, 245, 0.8)",
                    border_width=1,
                    border_color="#ccc",
                    textstyle_opts=opts.TextStyleOpts(color="#000",font_size=10),
                ),
                axispointer_opts=opts.AxisPointerOpts(
                    is_show=True,
                    link=[{"xAxisIndex": "all"}],
                    label=opts.LabelOpts(background_color="#777"),
                ),
                brush_opts=opts.BrushOpts(
                    x_axis_index="all",
                    brush_link="all",
                    out_of_brush={"colorAlpha": 0.1},
                    brush_type="lineX",
                ),
            )
            fname = '{}{}.html'.format(cfg.PATH_ANAFILE,'macro')
            linechart.render(fname)
        else:
            pass

        fname = '{}{}.html'.format(cfg.PATH_ANAFILE,'macro')
        linechart.render(fname) if renderfile else None
        self.macrochart = linechart #输出至可访问对象属性中
        return self

    def corelationAnalysis(self,str_reportName='',tpl_dateindex=None,tpl_baselist=None,tpl_targetlist=None):
        fname = '{}{}_corelation.csv'.format(cfg.PATH_ANAFILE,str_reportName)

        # scope up all target trade dates into a list
        if tpl_dateindex is None:
            return

        # define benchmarking securities, can be indice or stock
        if tpl_baselist is None:
            tpl_baselist = ('000001.SH','000300.SH','000905.SH','399006.SZ')
        # specify the target securities to compare with benchmarks
        if tpl_targetlist is None:
            tpl_targetlist = tuple(self.da.df_stockTbl['ts_code'])

        # populate all possible pairs specified by the benchmark and target scope
        analyticScope = tuple(product(tpl_baselist,tpl_targetlist))
        print('{} pairs are in scope for analysis'.format(len(analyticScope)))

        # specify result summary table structure
        df_result = pd.DataFrame(columns=['str_targetcode','str_basecode','sampleSize','validSampleRatio','CorelatePerc'])

        str_basecode = ''
        df_base = pd.DataFrame()

        #define benchmarking price, can be against 'high', 'low', 'open', 'close'
        str_price = 'close'
        for pair in analyticScope:
            # 先读取基准证券/指数的数据，循环中发现新的基准证券则对次变量重新赋值，如基准未变则只需将比较标的变量重新赋值
            if str_basecode != pair[0]:
                str_basecode,str_targetcode = pair
                df_base = self.da.query('load_alldaily', enumerate=False, tpl_dateindex=tpl_dateindex, ts_code=str_basecode)
                if len(df_base)<1:
                    continue
                # 计算出每行数据和上行数据的差值
                df_base['delta_'+str_price]=df_base[str_price]-df_base[str_price].shift(1)
                df_base['delta_'+str_price].fillna(value=0,inplace=True)
            else:
                str_targetcode = pair[1]

            df_target = self.da.query('load_alldaily', enumerate=False, tpl_dateindex=tpl_dateindex, ts_code=str_targetcode)

            if len(df_target)<1:
                continue

            # 根据之前制定的价格种类进行差价计算
            df_target = DataAgent.fq(df_target)
            df_target['delta_'+str_price]=df_target['adj_'+str_price]-df_target['adj_'+str_price].shift(1)
            df_target['delta_'+str_price].fillna(value=0,inplace=True)

            df_comb = pd.DataFrame.merge(df_base, right=df_target, how='left',left_on='trade_date', right_on='trade_date',suffixes=['_base','_target'])
            # 如base和target的delta乘积为正数说明正相关，反之则负相关
            df_comb['corelation'] = df_comb.apply(lambda x: 1 if x['delta_'+str_price+'_base']*x['delta_'+str_price+'_target']>0 else (-1 if x['delta_'+str_price+'_base']*x['delta_'+str_price+'_target']<0 else 0), axis=1)

            # 测量有效数据占比
            sampleSize = len(df_comb)
            #数出有正，负相关结论的交易日个数， -1代表负相关，0代表无数据或无变化，1代表正相关
            corelationGrp = df_comb.groupby('corelation')['trade_date'].count()
            #所有样本总数减去无数据或无变化的样本数量即可得到有效样本数量
            validSampleSize = (sampleSize - corelationGrp.loc[0]) if 0 in corelationGrp.index  else sampleSize
            validSampleRatio = round(validSampleSize/sampleSize,2) if sampleSize!=0 else 0

            #正相关样本数量占比
            CorelateCount = corelationGrp.loc[1] if 1 in corelationGrp.index  else 0
            CorelatePerc = round(CorelateCount/validSampleSize,2) if validSampleSize!=0 else 0

            resultRow = [str_targetcode,str_basecode,sampleSize,validSampleRatio,CorelatePerc]
            df_result.loc[len(df_result)] = resultRow

            print('done pair:{}-{}'.format(str_basecode,str_targetcode))

        libs.df_csv(fname,(df_result,))
        print('Corelation analysis done and result saved to {}'.format(fname))

    @staticmethod
    def normByRange(lst_input,delta=None,int_start=0,int_end=1000,):
        length = abs(int_end - int_start)
        delta = np.nanmax(lst_input) - np.nanmin(lst_input) if delta==None else delta
        return map(lambda x: round(((x- np.nanmin(lst_input))*length/delta),0), lst_input)

if __name__ == "__main__" :
    '''--------------------- General Parameters Secion -----------------'''
    an = Analytics(DataAgent(liveNet=False))
    dat_cursor = datetime(2019,4,16,15,1,tzinfo=timezone(cfg.STR_TIMEZONE))
    starttime = datetime.now().astimezone(timezone(cfg.STR_TIMEZONE))
    closestTradeDate = an.da.int_findClosestTradeDate(datetime.strftime(starttime,'%Y%m%d') )
    '''--------------------- General Parameters Secion -----------------'''

    '''--------------------- correlation analysis secion -----------------'''
    # int_startbound,int_endbound = 20180101,20190115
    # tpl_dateindex = tuple(str(i) for i in da.df_calTbl['cal_date'] if i>=int_startbound and i<=int_endbound)
    tpl_dateindex = tuple(str(i) for i in an.da.df_calTbl.loc[an.da.df_calTbl['cal_date'] <= closestTradeDate]['cal_date'][-30:] )
    tpl_baselist = ('000300.SH','399006.SZ')
    tpl_targetlist = tuple(an.da.df_stockTbl['ts_code'])
    # an.corelationAnalysis(str_reportName='stock',tpl_dateindex=tpl_dateindex,tpl_baselist=tpl_baselist,tpl_targetlist=tpl_targetlist)
    '''----------------- correlation analysis secion ends ----------------'''

    '''--------------------- Candle chart generation secion -----------------'''
    # int_startbound,int_endbound = 20180101,20190115
    # tpl_dateindex = tuple(str(i) for i in da.df_calTbl['cal_date'] if i>=int_startbound and i<=int_endbound)
    # tpl_dateindex = tuple(str(i) for i in da.df_calTbl.loc[da.df_calTbl['cal_date'] <= closestTradeDate]['cal_date'][-250:] )
    # an.candlechart('399006',subset='masterCandle',dat_cursor=None, MA=[5,10,21,60],renderfile=True)
    '''----------------- Cadle chart generation secion ends ----------------'''

    '''--------------------- Poll Msrket Summary secion -----------------'''
    # da.resampleStore(dat_cursor,tpl_targetlist=None)
    # an.pollsummary(renderfile=True)
    # libs.df_csv(cfg.PATH_BUGFILE,(df_stockBeatRollup,))
    '''----------------- Poll Msrket Summary secion ends ----------------'''

    '''--------------------- Macro Analysis -----------------'''
    # an.corelationMacro(lst_tscode=['000001.SH','000001.SZ'],lst_macro=['shibor.1w','pimon.cpi'],lst_findmedian=['pe',('000001.SZ',)],renderfile=True)
    '''--------------------- Macro Analysis ends -----------------'''


import json
import logging
import socket

import survillo.components.servos.micro_servo_sg90 as sg90
import survillo.components.servos.micro_servo_ts90a as ts90a
import survillo.crane.controlled_servo as cs

_IP = '0.0.0.0'
_PORT = 10003
_HORIZONTAL_SERVO_PIN = 17
_VERTICAL_SERVO_PIN = 27

_logger = logging.getLogger(__name__)

class ServoServer:

    def __init__(self, horizontal_servo, vertical_servo):
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket:
            _logger.debug('Server socket created')

            server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            server_socket.bind((_IP, _PORT))
            server_socket.listen(1)

            while True:
                _logger.debug('Server socket waiting for incoming connections')

                client_socket, from_address = server_socket.accept()
                with client_socket:
                    _logger.debug('Connection established with {}'.format(from_address))

                    while True:
                        data = client_socket.recv(1024)

                        if not data:
                            break

                        shutdown = False

                        msg = data.decode('utf-8')
                        for split_msg in msg.strip().split():

                            if not split_msg:
                                _logger.warning('Empty string received from server')
                                continue

                            _logger.debug('Command from client: {}'.format(split_msg))

                            if split_msg == 'q':
                                # Shut down the servos.
                                horizontal_servo.set_position(None)
                                vertical_servo.set_position(None)
                                shutdown = True
                                break

                            try:
                                json_message = json.loads(split_msg)
                            except json.JSONDecodeError as err:
                                _logger.error('Could not json-decode: {}'.format(split_msg))
                                continue

                            position_x = float(json_message.get('px'))
                            position_y = float(json_message.get('py'))
                            if position_x is None or position_y is None:
                                _logger.warning('String does not match expected pattern: {}'.format(split_msg))
                                continue

                            # The horizontal servo is upside down.
                            horizontal_servo.set_position(normalized_position=-position_x)
                            vertical_servo.set_position(normalized_position=position_y)

                        if shutdown:
                            break

                    _logger.debug('Closing client socket')

def main():
    horizontal_servo = cs.ControlledServo(pin=_HORIZONTAL_SERVO_PIN,
                                          min_pulse_width_s=sg90.MIN_PULSE_WIDTH_s,
                                          max_pulse_width_s=sg90.MAX_PULSE_WIDTH_s,
                                          frame_width_s=sg90.FRAME_WIDTH_s,
                                          nominal_max_angle_deg=sg90.MAX_ANGLE_deg,
                                          range_deg=None,
                                          default_angle_deg=0.0)

    vertical_servo = cs.ControlledServo(pin=_VERTICAL_SERVO_PIN,
                                        min_pulse_width_s=ts90a.MIN_PULSE_WIDTH_s,
                                        max_pulse_width_s=ts90a.MAX_PULSE_WIDTH_s,
                                        frame_width_s=ts90a.FRAME_WIDTH_s,
                                        nominal_max_angle_deg=ts90a.MAX_ANGLE_deg,
                                        range_deg=[-90, -60],
                                        default_angle_deg=-90)

    ServoServer(horizontal_servo=horizontal_servo, vertical_servo=vertical_servo)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main()


import pymongo
import re
from datetime import date, timedelta
# ------------------------------------------
from ToolPack import tools
from KeyGraph_Approach import common_phrases

def split_sentences(doc):
    p_sent_tokenizer = tools.load_pickle("/home/iraklis/PycharmProjects/AllTheNews/Pivot_Files/"
                                         "Classifiers/PunktSentenceTokenizer.pickle")
    sentences = p_sent_tokenizer.sentences_from_text(doc)
    return sentences

# Clean your documents with this method before searching for the noun phrases
def document_cleaning(doc_sentences):
    stop_words = common_phrases.PhraseDetection.load_stopwords()
    documents = list()
    doc_range = list()
    index = 0
    doc_number = 0

    for line in doc_sentences:
        # line_to_lowercase = line.lower()
        clean_line = re.sub('[^A-Za-z0-9]+', ' ', line)     # line_to_lowercase
        documents.append(clean_line)
        index += 1
        doc_range.append(index)
        doc_number += 1

    documents = [doc.strip() for doc in documents]
    clean_documents = list()

    for doc in documents:
        clean_documents.append(' '.join([word for word in doc.split() if word not in stop_words]))

    documents = clean_documents[:]

    return documents

def gather_documents(c_week):
    # Getting data from the mongo database
    client = pymongo.MongoClient()
    # Database name is minedNews
    db = client.all_the_news

    path = "/home/iraklis/PycharmProjects/AllTheNews/KeyGraph_Approach/Week_docs_sentences/"
    week_clean_sents = list()
    with open(path + c_week + '_ids.txt', 'w') as week_file:

        for document in db[c_week].find({}, no_cursor_timeout=True):
            line_sentences = split_sentences(document["text"])
            for sent in line_sentences:
                week_file.write(sent + "\n")
            clean_sent = document_cleaning(line_sentences)
            week_clean_sents.append(clean_sent)
    tools.save_pickle(path + c_week + "sents_of_doc.pickle", week_clean_sents)

def gather_documents_per_day():
    # Getting data from the mongo database
    client = pymongo.MongoClient()
    # Database name is minedNews
    db = client.all_the_news

    path = "/home/iraklis/PycharmProjects/AllTheNews/KeyGraph_Approach/Day_doc_sentences/"
    start_day = date(2016, 1, 4)
    stop_day = date(2017, 1, 1)
    c_day = start_day
    c_week = str(start_day.isocalendar()[0]) + "-" + str(start_day.isocalendar()[1])

    while c_day < stop_day:
        day_clean_sents = list()
        current_date = str(c_day.year) + "-" + str(c_day.month) + "-" + str(c_day.day)
        print(current_date)
        with open(path + current_date + '_ids.txt', 'w') as week_file:
            for document in db[c_week].find({"date": current_date}, no_cursor_timeout=True):
                line_sentences = split_sentences(document["text"])
                for sent in line_sentences:
                    week_file.write(sent + "\n")
                clean_sent = document_cleaning(line_sentences)
                if clean_sent:
                    day_clean_sents.append(clean_sent)
                else:
                    print("Got one on:", current_date, " ", c_week)
                    print(document["_id"])
        c_day += timedelta(1)
        c_week = str(c_day.isocalendar()[0]) + "-" + str(c_day.isocalendar()[1])
        tools.save_pickle(path + current_date + "_sents_of_doc.pickle", day_clean_sents)

if __name__ == "__main__":
    # gather_documents_per_day()
    a = tools.load_pickle("/home/iraklis/PycharmProjects/AllTheNews/KeyGraph_Approach/"
                          "Day_doc_sentences/2016-1-4_sents_of_doc.pickle")
    print()


import numpy as np
from shapely.ops import transform
import os
from tempfile import NamedTemporaryFile
from shapely.geometry import shape, box
from pyveda.utils import from_bounds
import numpy as np
from skimage.draw import polygon
from skimage.io import imread

class NDImageHandler(object):
    _default_dtype = np.float32

    @staticmethod
    def _payload_handler(*args, **kwargs):
        return NDImageHandler._bytes_to_array(*args, **kwargs)

    @staticmethod
    def _on_fail(shape=(3, 256, 256), dtype=np.uint8):
        return np.zeros(shape, dtype=dtype)

    @staticmethod
    def _bytes_to_array(bstring):
        if bstring is None:
            return on_fail()
        try:
            fd = NamedTemporaryFile(prefix='veda', suffix='.tif', delete=False)
            fd.file.write(bstring)
            fd.file.flush()
            fd.close()
            arr = imread(fd.name)
            if len(arr.shape) == 3:
                arr = np.rollaxis(arr, 2, 0)
            else:
                arr = np.expand_dims(arr, axis=0)
        except Exception as e:
            arr = NDImageHandler._on_fail()
        finally:
            fd.close()
            os.remove(fd.name)
        return arr

class BaseLabelHandler(object):
    @staticmethod
    def _get_transform(bounds, height, width):
        return from_bounds(*bounds, width, height)

    @staticmethod
    def _parse_response(res):
        return res['properties']['label']

    @staticmethod
    def _payload_handler(*args, **kwargs):
        raise NotImplementedError

class ClassificationHandler(BaseLabelHandler):
    _default_dtype = np.uint8

    @staticmethod
    def _payload_handler(item, klasses=[], **kwargs):
        payload = ClassificationHandler._parse_response(item)
        return [payload[klass] for klass in klasses]

class SegmentationHandler(BaseLabelHandler):
    _default_dtype = np.float32

    @staticmethod
    def _payload_handler(*args, **kwargs):
        return SegmentationHandler._handle_pixel_payload(*args, **kwargs)

    @staticmethod
    def _handle_pixel_payload(item, klasses=[], out_shape=None, **kwargs):
        payload = SegmentationHandler._parse_response(item)
        if len(out_shape) == 3:
            out_shape = out_shape[-2:]
        out_array = np.zeros(out_shape)
        value = 1
        for klass in klasses:
            shapes = payload[klass]
            try:
                out_array += rasterize(((shape(g), value) for g in shapes), out_shape=out_shape)
            except Exception as e:
                pass
        return out_array

    @staticmethod
    def _handle_geo_payload(item, imshape):
        out_shape = imshape
        if len(imshape) == 3:
            out_shape = imshape[-2:]
        out_array = np.zeros(out_shape)
        value = 1
        for k, features in item['data']['label'].items():
            try:
                out_array += SegmentationHandler._create_mask(features, value, out_shape)
                value += 1
            except Exception as e: # I think this is ValueError from rasterio but need check
                pass
        return out_array

    @staticmethod
    def _create_mask(shapes, value, _shape):
        mask = np.zeros(_shape, dtype=np.uint8)
        for f in shapes:
            coords = f['coordinates'][0]
            r, c = zip(*[(x,y) for x,y in coords])
            rr, cc = polygon(np.array(r), np.array(c))
            mask[rr, cc] = value

class ObjDetectionHandler(BaseLabelHandler):
    _default_dtype = np.float32

    @staticmethod
    def _payload_handler(*args, **kwargs):
        return ObjDetectionHandler._handle_pixel_payload(*args, **kwargs)

    @staticmethod
    def _handle_pixel_payload(item, klasses=[], out_shape=None, **kwargs):
        payload = ObjDetectionHandler._parse_response(item)
        return [payload[klass] for klass in klasses]

    @staticmethod
    def _handle_geo_payload(item, imshape):
        out_shape = imshape
        if len(imshape) == 3:
            out_shape = imshape[-2:]
        xfm = BaseLabelHandler._get_transform(item['data']['bounds'], *out_shape)
        labels = []
        for k, features in item['data']['label'].items():
            class_labels = []
            for i, f in enumerate(features):
                b = shape(f).bounds
                ll, ur = ~xfm * (b[0],b[1]), ~xfm * (b[2],b[3])
                class_labels.append([*ll, *ur])
            labels.append(class_labels)
        return labels


import json
import os
import sys
from string import Template

import sublime

from .lib import plistlib

sublime_settings = "Preferences.sublime-settings"
scope_name = "colored.comments.color."
MSG = Template(
    """
Would you like to change your color scheme to '$scheme'?
To permanently disable this prompt, set 'prompt_new_color_scheme'
to false in the Colored Comments settings."""
)

sublime_default_cs = [
    "Mariana.sublime-color-scheme",
    "Celeste.sublime-color-scheme",
    "Monokai.sublime-color-scheme",
    "Breakers.sublime-color-schem",
    "Sixteen.sublime-color-scheme",
]

class ColorManager:
    def __init__(self, new_color_scheme_path, tags, view, settings, regenerate, log):
        self.new_color_scheme_path = new_color_scheme_path
        self.view = view
        self.sublime_pref = None
        self.tags = tags
        self.settings = settings
        self.regenerate = regenerate
        self.log = log
        self.color_scheme = str()
        self.update_preferences = True
        self.awaiting_feedback = False

    def get_update_pref(self):
        return self.update_preferences

    def get_awaiting_feedback(self):
        return self.awaiting_feedback

    def set_awaiting_feedback(self, status):
        self.awaiting_feedback = status

    def _add_colors_to_scheme(self, color_scheme, is_json):
        scheme_rule_key = "rules" if is_json else "settings"
        settings = color_scheme[scheme_rule_key]
        scope_exist = bool()
        updates = bool()

        for tag in self.tags:
            curr_tag = self.tags[tag]
            if not curr_tag.get("color", False):
                continue

            name = _get_color_property("name", curr_tag)
            background = _get_color_property("background", curr_tag)
            foreground = _get_color_property("foreground", curr_tag)
            if False in [name, background, foreground]:
                continue

            scope = "{}{}".format(scope_name, name.lower().replace(" ", "."))

            for setting in settings:
                if "scope" in setting and setting["scope"] == scope:
                    scope_exist = True

            if not scope_exist:
                updates = True
                entry = dict()
                entry["name"] = "[Colored Comments] {}".format(name.title())
                entry["scope"] = scope
                if is_json:
                    entry["foreground"] = foreground
                    entry["background"] = background
                else:
                    entry["settings"] = dict()
                    entry["settings"]["foreground"] = foreground
                    entry["settings"]["background"] = background

                settings.append(entry)
        color_scheme[scheme_rule_key] = settings
        return updates, color_scheme

    def _create_custom_color_scheme_directory(self):
        path = os.path.join(sublime.packages_path(), self.new_color_scheme_path)
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def create_user_custom_theme(self):
        if self.awaiting_feedback:
            return
        self.awaiting_feedback = True
        if not self.tags:
            self.awaiting_feedback = False
            return

        self.sublime_pref = sublime.load_settings(sublime_settings)
        color_scheme = self.sublime_pref.get("color_scheme")
        if self.regenerate and self.settings.get("old_color_scheme", "") != "":
            color_scheme = self.settings.get("old_color_scheme", "")

        self.settings.set("old_color_scheme", color_scheme)
        sublime.save_settings("colored_comments.sublime-settings")
        cs_base = os.path.basename(color_scheme)

        if cs_base[0:16] != "Colored Comments":
            cs_base = "{}{}".format("Colored Comments-", cs_base)

        custom_color_base = self._create_custom_color_scheme_directory()
        new_cs_absolute = os.path.join(custom_color_base, cs_base)
        self.color_scheme = "{}{}{}{}".format(
            "Packages/", self.new_color_scheme_path, "/", cs_base
        )
        print(self.color_scheme)

        updates, loaded_scheme, is_json = self.load_color_scheme(color_scheme)

        if self.regenerate or updates or color_scheme != self.color_scheme:
            try:
                os.remove(new_cs_absolute)
            except OSError as ex:
                self.log.debug(str(ex))
                pass
            if is_json:
                with open(new_cs_absolute, "w") as outfile:
                    json.dump(loaded_scheme, outfile, indent=4)
            else:
                with open(new_cs_absolute, "wb") as outfile:
                    outfile.write(plistlib.dumps(loaded_scheme))

        if color_scheme != self.color_scheme:
            if sublime.ok_cancel_dialog(
                MSG.substitute(scheme=self.color_scheme), "Confirm"
            ):
                self.sublime_pref.set("color_scheme", self.color_scheme)
                sublime.save_settings("Preferences.sublime-settings")
                self.settings.set("prompt_new_color_scheme", False)
                sublime.save_settings("colored_comments.sublime-settings")
            self.update_preferences = False
            self.awaiting_feedback = False

    def load_color_scheme(self, scheme):
        is_json = bool()
        try:
            if scheme in sublime_default_cs:
                scheme = "{}{}".format("Packages/Color Scheme - Default/", scheme)
            scheme_content = sublime.load_binary_resource(scheme)
        except Exception as ex:
            sublime.error_message(
                " ".join(
                    [
                        "An error occured while reading color",
                        "scheme file. Please check the console",
                        "for details.",
                    ]
                )
            )
            self.log.debug(
                "[Colored Comments]: {} - {}".format(
                    self.load_color_scheme.__name__, ex
                )
            )
            raise
        if scheme.endswith(".sublime-color-scheme"):
            is_json = True
            updates, color_scheme = self._add_colors_to_scheme(
                sublime.decode_value(scheme_content.decode("utf-8")), is_json
            )
        elif scheme.endswith(".tmTheme"):
            updates, color_scheme = self._add_colors_to_scheme(
                plistlib.loads(bytes(scheme_content)), is_json
            )
        else:
            sys.exit(1)
        return updates, color_scheme, is_json

def _get_color_property(property, tags):
    if not tags["color"].get(property, False):
        return False
    return tags["color"][property]


from __future__ import print_function
import urllib2, urllib
from bs4 import BeautifulSoup, Comment
import os
import glob

liveFileDir = '/wiki/liveArticles/'  # dir of your live Wikipedia articles downloaded from scrapeWikiVital.py
staticFileDir = '/VOLUMES/LIBRARYBOX/static-Wikipedia-100'  # dir of where you want the static copies of the Wikipedia articles
mediaFileDir = staticFileDir+'/wiki/en/'  # the static articles will be placed within a /wiki/en/ for the English versions

if not os.path.exists(mediaFileDir):
		try:
			os.makedirs(mediaFileDir)
		except:
			pass

def convertWikiArticle(filename):
	global liveFileDir, staticFileDir, mediaFileDir
	frenchlink = ''

	with open(filename,'r') as f:
		s = f.read()

	articleName = filename
	articleName = urllib.unquote(articleName).decode('utf8').strip()
	articleName = articleName.replace('.'+liveFileDir,'').strip()
	print(articleName)

	soup = BeautifulSoup(s)

	for div in soup.findAll('table', { 'class' : 'navbox' }):
		div.extract()

	for div in soup.findAll('table', { 'class' : 'persondata' }):
		div.extract()

	for div in soup.findAll('div', { 'class' : 'noprint' }):
		div.extract()

	for div in soup.findAll('ul', { 'class' : 'noprint' }):
		div.extract()

	#for div in soup.findAll('div', { 'class' : 'thumb' }):
	#	div.extract()

	for div in soup.findAll('div', { 'class' : 'hatnote' }):
		div.extract()

	for div in soup.findAll('div', { 'class' : 'after-portlet' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'mw-head' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'p-logo' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'catlinks' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'p-interaction' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'p-tb' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'siteNotice' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'jump-to-nav' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'p-coll-print_export' }):
		div.extract()

	for div in soup.findAll('div', { 'id' : 'section_SpokenWikipedia' }):
		div.extract()

	for div in soup.findAll('div', { 'class' : 'ns-0' }):
		div.extract()

	for div in soup.findAll('div', { 'class' : 'topicon' }):
		div.extract()

	for div in soup.findAll('ul', { 'id' : 'footer-places' }):
		div.extract()

	for div in soup.findAll('link', { 'rel' : 'dns-prefetch' }):
		div.extract()

	for div in soup.findAll('link', { 'rel' : 'stylesheet' }):
		div.extract()

	#for div in soup.findAll('table', { 'class' : 'plainlinks' }):
	#	div.extract()

	for div in soup.findAll('script'):
		div.extract()

	for div in soup.findAll('style'):
		div.extract()

	for div in soup.findAll('span', { 'class' : 'mw-editsection' }):
		div.extract()

	comments = soup.findAll(text=lambda text:isinstance(text, Comment))
	[comment.extract() for comment in comments]

	styletag = soup.new_tag('link')
	styletag.attrs['rel'] = 'stylesheet'
	styletag.attrs['href'] = '../css/style.css'
	soup.head.append(styletag)

	for div in soup.findAll('a', { 'class' : 'external text'}):
		div['target'] = '_blank'

	for div in soup.findAll('a', { 'dir' : 'ltr'}):
		#div.unwrap()
		pass

	for div in soup.findAll('a', { 'lang' : 'fr' , 'hreflang' : 'fr'}):
		frenchlink = div['href']
		print('FrenchLink: '+frenchlink)  # get the French link for the article to download, and write it to a file to get later

		if frenchlink != '':
			with open("lists/listFrenchArticles2Download.txt", "a") as myfile:
				myfile.write('http:'+frenchlink+'\n')
		else:
			with open("lists/missingVitalFrenchArticles.txt", "a") as myfile:
				myfile.write(articleName+'\n')

	for div in soup.findAll('li', { 'id' : 'footer-info-copyright'}):
		div.clear()
		div.append('Text is available under the ')
		new_tag = soup.new_tag("a", href="Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License.html")
		new_tag.string = 'Creative Commons Attribution-ShareAlike License'
		div.append(new_tag)
		div.append('; additional copyright terms may apply to images, audio and video.  Please see this article on wikipedia.org for copyright information.')

	# Clear and then rebuild the navigation sidebar; needs to be rewritten in reverse order
	for div in soup.findAll('div', { 'id' : 'p-navigation' }):
		div.clear()

		new_tag = soup.new_tag('h3')
		new_tag.string = 'Search Wikipedia'
		div.append(new_tag)
		div.h3['id']='p-navigation-label'

		new_tag = soup.new_tag('input')
		#new_tag.string = 'Search'
		div.append(new_tag)
		div.input['id']='searchInput'
		div.input['type']='search'
		div.input['name']='search'
		div.input['accesskey']='f'
		div.input['placeholder']='Search'
		div.input['style']='margin-left:10px;margin-bottom:10px;width:130px;'

		new_tag = soup.new_tag('h3')
		new_tag.string = 'Navigation'
		div.append(new_tag)
		div.h3['id']='p-navigation-label'

		new_tag = soup.new_tag('div', { 'class' : "body" })
		div.append(new_tag)
		div.div['class']='body'

		new_tag = soup.new_tag('ul')
		div.div.append(new_tag)

		new_tag = soup.new_tag('li')
		div.div.ul.append(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Mathematics')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Mathematics.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Technology')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Technology.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Physical sciences')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Physical_sciences.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Biology and health sciences')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Biology_and_health_sciences.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Society and social sciences')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Society_and_social_sciences.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Everyday life')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Everyday_life.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Philosophy and religion')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Philosophy_and_religion.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Arts')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Arts.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('Geography')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_Geography.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('History')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_History.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = ('People')
		div.li.append(new_tag)
		div.li.a['href']='../en/Subject_People.html'

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = 'Main Page'
		div.li.append(new_tag)
		div.li.a['href']='../en/Main_Page.html'

	# Clear and then rebuild the language sidebar
	for div in soup.findAll('div', { 'id' : 'p-lang' }):
		div.clear()
		new_tag = soup.new_tag('h3')
		new_tag.string = 'Languages'
		div.append(new_tag)
		div.h3['id']='p-lang-label'
		new_tag = soup.new_tag('div', { 'class' : "body" })
		div.append(new_tag)
		div.div['class']='body'
		new_tag = soup.new_tag('ul')
		div.div.append(new_tag)

		new_tag = soup.new_tag('li')
		div.div.ul.append(new_tag)

		if frenchlink != '':
			new_tag = soup.new_tag('a')
			div.li.append(new_tag)
			div.li.a['href']='../fr/'+articleName  # needs to be changed to whatever the French title is
			new_tag.string = (u'Fran\u00E7ais')
		else:
			new_tag = soup.new_tag('span')

		new_tag = soup.new_tag('li')
		div.li.insert_before(new_tag)
		new_tag = soup.new_tag('a')
		new_tag.string = 'English'
		div.li.append(new_tag)
		div.li.a['href']='../en/'+articleName+'.html'

	for div in soup.find('div', { 'id' : 'mw-content-text' }).findAll('a'):

		hreflink = str(div['href'].encode('utf-8'))
		if hreflink[:1] != '#' and '/wiki/' in hreflink:
			div['href'] = div['href'].replace('/wiki/','../en/')
			div['href'] += '.html'

	try:
		for div in soup.find('div', { 'id' : 'mw-content-text' }).findAll('img'):

			imglink = str(div['src'].encode('utf-8'))
			print(imglink)
			#imglink = imglink.replace('//upload.wikimedia.org/wikipedia/','../')
			#print(imglink)
			if not '?' in imglink:
				with open("lists/listInfoBoxImgs.txt", "a") as myfile:
					myfile.write(imglink+'\n')

	except:
		pass

	try:
		for div in soup.find('div', { 'id' : 'mw-content-text' }).findAll('source'):

			ogglink = str(div['src'].encode('utf-8'))
			print(ogglink)
			#ogglink = ogglink.replace('//upload.wikimedia.org/wikipedia/','../')
			#print(imglink)

			with open("lists/listInfoBoxOggs.txt", "a") as myfile:
				myfile.write(ogglink+'\n')

	except:
		pass

	soup.prettify()

	new_soup = str(soup)
	new_soup = new_soup.replace('//upload.wikimedia.org/wikipedia/','../')

	try:
		path = staticFileDir  # I ran out of space on my laptop, so I'm just going to throw all these files on a USB
		f = open(path+'/wiki/en/'+articleName,'w')
		print(new_soup,file=f)
		f.close
	except:
		print('Static save failed!!!')

i=0
for infile in glob.glob( os.path.join('./wiki/liveArticles/', '*.html') ):
	i+=1
	print('Article #'+str(i))
	print(infile)
	convertWikiArticle(infile)