#!/usr/bin/env python3

VERSION="""grrv2_ocu_color.py
use with grrv2_rover_controller_2017-12-12.py
"""

LANC_COMMAND_HELP = """
LANC COMMAND HELP

KEYBOARD COMMANDS
?: this help message
P: PHOTO    //Doesn't work
R, r: REC   //To confirm if RECORDING, select 'M' for Menu. REC/STBY status is displayed
z: ZOOM_OUT_4
Z: ZOOM_IN_4
f: FOCUS_NEAR //WORKS, IN MANUAL FOCUS MODE ONLY (Set Manual/Auto via menu).
F: FOCUS_FAR  //WORKS, IN MANUAL FOCUS MODE ONLY

CAMCORDER MENU COMMANDS
M, m: MENU
<CR>: SELECT / EXECUTE
8: MENU_UP
2: MENU_DOWN
4: MENU_LEFT or PREV;
6: MENU_RIGHT or NEXT;
O: CAMERA POWER ON
o: CAMERA POWER OFF
L: VIDEO LIGHTS ON
l: VIDEO LIGHTS OFF

RAW LANC COMMAND MODE
X, x: rawHexCommandMenu(); Enter header byte, then raw hex number command byte.
	If you don't know what this means, don't mess with it. :-)
"""

import pygame
from pygame import *

import getopt
import math
import os
import string
import sys

from pygame.locals import *
from socket import *
from time import *
from time import sleep

import colorama
from colorama import Fore, Back, Style

colorama.init()

# Joystick Structure info:
# 8 floats. Up to 6 real axes and 2 virtual axes (from buttons). joystickx[0][0] to [0][7]
# 13 ints.  Up to 12 buttons. joystickx[1][0] to [1][12]. [1][0]
# is ignored since joysticks label 12 buttons 1-12, not 0-11
# 1 string. Joystick Name as pulled from Pygame joystickx[2]

jstick = {
	"axes": [0,0,0,0],
	"scale": [125, -125, 125, -125],	# Corrects native joystick orientation, motor polarity, etc
	"buttons": [0,] * 13,  				# 13 bytes for 12 buttons: 0 is ignored since joysticks number 1-12 not 0-11
	"hat": [0,0], 						# joystick hat x and y, -1, 0, 1
}

class UDPSocket:
	"""UDPSocket tracks a remote and simplifies handling multiple targets"""
	def __init__(self, remoteIP, remotePort):
		"""Opens a new socket with a default destination of remoteIP, remotePort

        Args:
            remoteIP (str): Default remote IP.
            remotePort (int): Default remote Port
        """

		self.remoteIP = remoteIP
		self.remotePort = remotePort
		self.sockobj = socket(AF_INET, SOCK_DGRAM)
		self.sockobj.settimeout(1.0)
		self.last_message = None

	def sendto(self, message, target = None):
		"""Send a message to target"""
		msg = bytes(message)
		if target is None:
			target = (self.remoteIP, self.remotePort)

		try:
			self.sockobj.sendto(msg, target)
		except Exception as e:
			print(e)

		if msg != self.last_message:
			with open("command.log", "wb+") as f:
				f.write(msg)
			self.last_message = msg

	def recvfrom(self, size):
		"""receives a message

		Returns: (data, remote_address)
		"""
		data, remote_addr = self.sockobj.recvfrom(size)
		self.last_remote_addr = remote_addr
		return data, remote_addr

def initalize_joysticks():
	"""Walks through the list of joysticks and ensures PYGame has initilized them"""
	grab_input()

	for x in range(joystick.get_count()): 			# inits however many joysticks are attached, usually 1 or 2
		j = joystick.Joystick(x) 					# Logitech Dual Action is 1 joystick w/ 2 sticks with 1 XY axis pair (4 axes).
		j.init()                       				# Most elaborate joysticks are 1 joystick 1 stick with 2 axes (x, y)
		txt = 'Enable Joystick: ' + j.get_name()
		# jstick['name'] = j[x].get_name()
		print(txt)

	release_input()

def setup_window():
	"""sets the x,y position of the window and required video mode flags
	we don't actually need to run the window to run the program
	"""
	os.environ['SDL_VIDEO_WINDOW_POS'] = "%d, %d" % (1600, 40)
	video_flags = OPENGL|DOUBLEBUF
	display.set_mode((128,96), video_flags)

def main():
	print(VERSION)

	# pygame initialization
	setup_window()
	pygame.init()
	clock, ticks = pygame.time.Clock(), pygame.time.get_ticks()
	initalize_joysticks()

	# a socket to talk to the rover
	rover_socket = UDPSocket('192.168.1.99', 60615)

	print('Starting' + "\n"*4)

	pilt_upcounter = 0
	pilt_downcounter = 255
	loops=1

	motor_enable_flag = 0

	last_keydown = -1
	keyboard_keys = [0,0]

	while True:
		clock.tick(300)  # clock.tick(x) x = Hertz of control loop. 100=100Hz, 20 = 20 Hz Works

		for e in event.get():
			# print e.type,'  ',

			if e.type == pygame.JOYBUTTONDOWN:
				button = e.dict['button'] + 1
				# print("button down")
				jstick['buttons'][0] = button
				if button == 9:
					grab_input()
				if button == 10:
					release_input()
				# print button, ' ',jstick['buttons'][button]

			if e.type == pygame.JOYBUTTONUP:
				# print("button up")
				button = e.dict['button'] + 1
				jstick['buttons'][0] = 0
				# print button, ' ',jstick['buttons'][button]

			if e.type == pygame.KEYDOWN:
				# print(e)
				key, mod, uni = e.dict['key'], e.dict['mod'], e.unicode
				if len(uni) > 0:
					uni = ord(uni)
				else:
					uni = 0
				# key = e.dict['key'] - 32 * e.dict['mod'] # mod = 1 is shift

				# check for control keys
				if mod & pygame.KMOD_CTRL:
					if key == pygame.K_l:		# lock mouse and keyboard to game window
						grab_input()

					if key in (pygame.K_c, pygame.K_d):
						end_program()

				if key == pygame.K_ESCAPE:		# release mouse and keyboard from game window
					release_input()

				if key == pygame.K_SPACE: 		# ALL STOP  SPACE BAR (Big, easy to hit in a panic for faux E-STOP)
					motor_enable_flag = 0

				if key == pygame.K_e and mod & pygame.KMOD_SHIFT: # 'E', ENABLE MOTORS
					motor_enable_flag = 1

				# map arrows, pygame's 273..276 (up, down, right, left) => 128..131
				if key in (pygame.K_LEFT, pygame.K_RIGHT, pygame.K_UP, pygame.K_DOWN):
					map = {pygame.K_LEFT:131, pygame.K_RIGHT:130, pygame.K_UP:128, pygame.K_DOWN:129}
					keyboard_keys[0] = map[key]

				if key == 47:
					print(LANC_COMMAND_HELP)

				# Only send non-continuous commands once
				continuous_key_commands = ('f','F','z','Z')
				if uni not in continuous_key_commands and uni == last_keydown:
					uni = -1
					keyboard_keys[0] = 0

				# The above, including local commands & esc, space, and E all fall through to here and will be sent to the remote. Intentional?
				if uni >= 0 and uni < 256:
					keyboard_keys[0] = uni

			if e.type == pygame.KEYUP:
				key, mod = e.dict['key'], e.dict['mod']

				if key >= 0 and key < 256:
					keyboard_keys[0] = 0

				if key in (pygame.K_LEFT, pygame.K_RIGHT, pygame.K_UP, pygame.K_DOWN):
					keyboard_keys[0] = 0

				last_keydown = -1

			if e.type == pygame.JOYAXISMOTION:
				axis = e.dict['axis']
				# print("joy motion")
				jstick['axes'][axis] = int(e.dict['value'] * jstick['scale'][axis]) # -1.0 to 0 to 1.0 * +/-125

			if e.type == pygame.JOYHATMOTION: # HAT MOTION Use hat +/-y for LANC zoom? And +/-x for focus?
				# print("joy hat motion")
				jstick['hat'] = e.dict['value'] # 0,1 are x,y
				# print("\njstick hat: ", jstick['hat'])
				# since all LANC commands must be sent down the serial LANC wire
				# it's best to treat the Hat as keystrokes so they never compete
				# and the keystroke SwitchCase can prioritize them

				focus_commands = { 1: ord('F'), -1: ord('f') }
				zoom_commands =  { 1: ord('Z'), -1: ord('z') }

				# look it up in the dictionary. If it's not there, use '0'
				key = focus_commands.get(jstick['hat'][0], 0)
				# look up zoom state, if any, and allow that to override focus
				key = zoom_commands.get(jstick['hat'][1], key)
				# print('jstick hat to key', key)
				keyboard_keys[0] = key

			# if e.type == pygame.JOYBALLMOTION:
			# 	pass
			# if e.type in (pygame.MOUSEBUTTONDOWN, pygame.MOUSEBUTTONUP, pygame.MOUSEMOTION):
			# 	pass

		# send the joystick and keyboard commands to the rover
		motions = stick_to_motions(jstick['axes'])
		# print(motions, "\n")
		msg = build_rover_command_byte_packet(motions, joystick, keyboard_keys, motor_enable_flag)
		print('cmd_chr', *msg, end = ' ')
		rover_socket.sendto(msg)

		# RECEIVE DATA FROM ROBOT
		try:
			rover_packet, remote_address = rover_socket.recvfrom(1024)
			rover_status, pilt_status, rover_message = unpack_rover_message(rover_packet)

			pilt_upcounter = (pilt_upcounter + 1) % 255
			pilt_downcounter = 255 - pilt_upcounter

			print_rover_pilt_status(rover_status, pilt_status, rover_message, pilt_upcounter, pilt_downcounter, loops)
		except timeout:
			print('NO ROVER STATUS PACKET RECEIVED.  Last known ', end=' ')

		print("                \r", end=' ')
		sys.stdout.flush()

		loops = loops + 1
		if loops == 101:
			loops = 1

def pilt_indicator_hue(x):
	"""return a color format for printing based on the range of the value"""

	# Well within the limits of travel of pan and tilt
	if 96 <= x <= 160:
		return Style.BRIGHT + Fore.GREEN

	# Approaching the limits of travel of pan and tilt
	if 64 <= x <= 192:
		return Style.BRIGHT + Fore.YELLOW

	# AT OR EXCEEDING the limits of travel of pan and tilt
	return Style.BRIGHT + Fore.RED

def pilt_indicator(x):
	"""Convert a PILT reading to a colorized string"""
	return pilt_indicator_hue(x) + str(x) + Style.RESET_ALL

def print_rover_pilt_status(rover_status, pilt_status, rover_message, pilt_upcounter, pilt_downcounter, loops):
	"""Prints one line corresponding to the rover status. Multiple calls will overwrite previous lines."""
	print(' rover_status', *rover_status, end=' ')

	print('   pilt_status', end=' ')

	for i in range(min(14, len(pilt_status))):
		if i==2:
			print(pilt_indicator(pilt_upcounter), end = ' ')
		elif i==5:
			print(pilt_indicator(pilt_downcounter), end = ' ')
		else:
			print(pilt_status[i], end=' ')

	print('  %s   %d            \r' % (rover_message, loops), end=' ')

	sys.stdout.flush()

def build_rover_command_byte_packet(motions, joystick, keyboard_keys, motor_enable_flag):
	"""Builds a rover command message from parameters"""

	def wire_signed(x):
		"""Converts (-1,1) to (0,1) for PILT"""
		if x == -1:
			return 0
		return 1

	msg = [
		motions['pwm_left'],						# Left Stick (X-Y mixed): Left PWM abs value
		wire_signed(motions['sign_left']),			# Left Stick (X-Y mixed): Left PWM boolean sign map(1=1, -1=0)
		motions['pwm_right'],						# Left Stick (X-Y mixed): Right PWM abs value
		wire_signed(motions['sign_right']),			# Left Stick (X-Y mixed): Right PWM boolean sign map(1=1, -1=0)
		abs(motions['stick_pan']),					# Right Stick,PWM, Pan
		wire_signed(sign(motions['stick_pan'])),	# Right Stick,DIR, Pan
		abs(motions['stick_tilt']),					# Right Stick,PWM, Tilt
		wire_signed(sign(motions['stick_tilt'])),	# Right Stick,DIR, Pan
		jstick['buttons'][0],
		keyboard_keys[0], 				# commands for lanc, lights, whatever to Rover
		jstick['hat'][0] + 1,	 		# JS Hat values are -1, 0, +1.
		jstick['hat'][1] + 1, 			# be unsigned: like joystick axes, it must be converted
		motor_enable_flag,
		0, 								# Spare
		0, 								# Spare
		0, 								# Spare
	] + [45]
	# [45] is an end of message marker

	assert len(msg) == 17
	return msg

def unpack_rover_message(rover_packet):
	"""break apart the received packet and convert from array of bytes to native python types
	Returns:
		rover_status (:obj:`list` of :obj:`int`))
		pilt_status  (:obj:`list` of :obj:`int`))
		rover_message (str)
	"""

	rover_status  = [x for x in rover_packet[0:16]]	# 16 bytes rover
	pilt_status   = [x for x in rover_packet[16:30]]	# 14 bytes Arduino status
	rover_message = rover_packet[32:48]						# Arduino puts.
	if not isinstance(rover_message, str):
		# In Python 3 strings are no longer equivalent to byte arrays, so go through a decode step
		rover_message = rover_message.decode('ascii')
	return rover_status, pilt_status, rover_message

def stick_to_motions(joystick):
	"""Translates joystick to PWMs, signs, octant, pan, and tilt"""
	pwm_left, pwm_right, sign_left, sign_right, octant = map_x_y_to_left_right_pwms(jstick['axes'])
	stick_pan = jstick['axes'][2]
	stick_tilt = jstick['axes'][3]
	motions = {
		"pwm_left": pwm_left,
		"pwm_right": pwm_right,
		"sign_left": sign_left,
		"sign_right": sign_right,
		"stick_pan": stick_pan,
		"stick_tilt": stick_tilt,
		"octant": octant,
	}
	return motions

def get_octant(joystick):
	"""get_octant() handles left gamepad joystick mixing for the tank skid-steering
	As normal cartesian grid has 4 quadrants where X and Y are + /-, we need 8 divisions
	So 'octants'. Based on 3 conditions:
		abs_x => abs_y; sgn_x >0; sgn_y >0. from these are derived the left and right
		'tank track' pwm mixing for single stick skid steer used in map_x_y_to_left_right()
	"""

	stick_x = jstick['axes'][0]
	stick_y = jstick['axes'][1]

	octant = 0
	if abs(stick_y) >= abs(stick_x):
		octant += 4

	if stick_y < 0:
		octant += 2

	if stick_x < 0:
		octant += 1

	return octant

def map_x_y_to_left_right_pwms(joystick):
	"""basically, left joystick positions within XY octants defined as:
	stick fwd, left right tracks fwd: back = bkwd
	stick 90 left or right = pivot lft or rt (one track fwd opposite track rev)
	stick 45 deg = soft turn; one track moving, other track stopped.
	intermediate angles smooth continuum of translate rotated between the above
	"""

	octant = get_octant(jstick['axes'])

	stick_x, stick_y = jstick['axes'][0], jstick['axes'][1]
	abs_x, abs_y     = abs(stick_x), abs(stick_y)
	sign_x, sign_y   = sign(stick_x), sign(stick_y)

	if octant in (0, 3):
		pwm_left = abs_y
		pwm_right = abs_y - abs_x
		sign_left = sign_y
		sign_right = sign_y

	if octant in (1, 2):
		pwm_left = abs_y - abs_x
		pwm_right = abs_y
		sign_left = sign_y
		sign_right = sign_y

	if octant in (4, 7):
		pwm_left = abs_x
		pwm_right = abs_x - abs_y
		sign_left = sign_x
		sign_right = -sign_x

	if octant in (5, 6):
		pwm_left = abs_x - abs_y
		pwm_right = abs_x
		sign_left = sign_x
		sign_right = -sign_x

	dead_boundary = 12

	if pwm_left < dead_boundary:
		pwm_left = 0
	if pwm_right < dead_boundary:
		pwm_right = 0

	return (pwm_left, pwm_right, sign_left, sign_right, octant)

def sign(number):
	"""Returns 1 if the number >= 0, otherwise -1"""
	if number >= 0:
		return 1
	return -1

def grab_input():
	"""Lock mouse and keyboard to window"""
	event.set_grab(True)

def release_input():
	"""Release mouse and keyboard from being locked to window"""
	event.set_grab(False)

def end_program():
	release_input()
	print("ALL STOP")
	print("EXITING OCU PROGRAM")
	sys.exit(0) # exits program

if __name__ == '__main__':
	main()


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 12 14:07:57 2020

@author: USER
"""

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
import re

link = 'https://hh.ru'
text = 'инженер электрик'
area_num = 2

def page_data(soup):
    #dataframe
    vacancies = pd.DataFrame()
    #блок со всеми вакансиями листа
    vac_block = soup.find('div', {'class':'vacancy-serp'})
    #список вакансий
    vac_list = vac_block.find_all('div', {'class':'vacancy-serp-item'}, recurcive=False)
    #данные по каждой вакансии
    for vac in vac_list:
        vac_dict = {}
        vac_dict['name'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-title'}).text
        vac_dict['salary'] = vac.find('div', {'class':'vacancy-serp-item__sidebar'}).text
        if '-' in vac_dict['salary']:
            res = re.search(r'([\d ]+\D*[\d ]*)-([\d ]+\D*[\d ]*)',vac_dict['salary'])
            vac_dict['min_salary'] = res.group(1)
            vac_dict['max_salary'] = res.group(2)
        elif 'от' in vac_dict['salary']:
            res = re.search(r'([\d]+\D*[\d]*) ',vac_dict['salary'])
            vac_dict['min_salary'] = res.group(1)

        elif 'до' in vac_dict['salary']:
            res = re.search(r'([\d]+\D*[\d]*) ',vac_dict['salary'])
            vac_dict['max_salary'] = res.group(1)

        else:
            None
        if vac_dict['salary']:
            vac_dict['currency'] = re.search(r' ([\w\.]+$)',vac_dict['salary']).group(1)
        else:
            None
        vac_dict['link'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-title'})['href']
        if vac.find('a', {'data-qa':'vacancy-serp__vacancy-employer'}):
            vac_dict['company'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-employer'}).text
        else:
            vac_dict['company']='not_found'
        vac_dict['city'] = vac.find('span', {'data-qa':'vacancy-serp__vacancy-address'}).text
        vac_dict['source'] = 'hh.ru'
        vacancies = vacancies.append(vac_dict, ignore_index = True)
    return(vacancies)

def scrap_hh(link, text, area_num):
    area = area_num
    vacancy_name = text
    url = link
    params = {'area':area,
              'st':'searchVacancy',
              'text': vacancy_name,
              'fromSearch':'true'}
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
    rq = requests.get(url+'/search/vacancy', params=params, headers=headers)
    soup = bs(rq.text, 'lxml')
    result = pd.DataFrame()
    result = page_data(soup)
    print(f'страница {1} добавлена')
    pager_block = soup.find('div', {'data-qa':'pager-block'})
    next_page_tail = pager_block.find('a', {'data-qa':'pager-next'})
    i  = 2
    while next_page_tail:
        soup = bs(requests.get(url+next_page_tail['href'], headers = headers).text, 'lxml')
        result = result.append(page_data(soup), ignore_index = True)
        pager_block = soup.find('div', {'data-qa':'pager-block'})
        next_page_tail = pager_block.find('a', {'data-qa':'pager-next'})
        print(f'страница {i} добавлена')
        i = i + 1
    return(result)

df = scrap_hh(link, text, area_num).loc[:, ['name','city', 'min_salary', 'max_salary', 'currency', 'link', 'source']]

print(df)


#!/usr/bin/python3
from PyQt5.QtCore import Qt,QTimer,QDateTime,QDate,QTime
from PyQt5.QtWidgets import (QApplication, QGridLayout, QLabel, QLineEdit, QMessageBox, QPushButton, QTextEdit, QWidget)
from PyQt5.QtGui import QPixmap
import math,sys,datetime

class Main_window(QWidget):
	def set_time(self):
		self.czas = datetime.datetime(2015,2,3,14,2)
		self.teraz = datetime.datetime.now()
		self.pozostaly=self.czas - self.teraz
		self.time_label.setText("Pozostało: "+str(self.pozostaly))

	def __init__(self):
		super().__init__()
		self.setWindowTitle("Zegar Śmierci")
		self.layout=QGridLayout()
		self.time_label=QLabel()
		self.layout.addWidget(self.time_label)
		self.setLayout(self.layout)
		self.timer=QTimer()
		self.timer.timeout.connect(self.set_time)
		self.timer.start(0)
		self.obrazek = QPixmap("Mentzen.png")
		self.mietek = QLabel()
		self.mietek.setPixmap(self.obrazek)
		self.layout.addWidget(self.mietek)

app = QApplication(sys.argv)
window = Main_window()
window.show()
sys.exit(app.exec())


#===== type hints ===========
from typing import *

x1 = 10  # type: int
# x2 = 10  # type: str

x3: int = 15

def my_plus(x: int, y: Optional[int]=None) -> int:
    if y is None:
        y = 88
    return x + y

print(my_plus(1, 2))
print(my_plus(2))
# print(my_plus("hello", "world"))

def g(x: Sequence[int]) -> None:
    print(len(x))
    print(x[1])
    for i in x:
        print(i)
    print()

g([1,2,3,4])
g((1,2,3,4))
# g('ssjlsjisjsij')

#===== defaultdict =====
from collections import defaultdict
from pprint import pprint
names = '''bob alice anna abba bot cat caro candy dom doc dog dot ab'''.split()
d = defaultdict(list)
for name in names:
    feature = name[0]
    d[feature].append(name)

pprint(d, depth=40)

pprint(sorted(names, key=len))


import numpy as np
from collections import OrderedDict

import torch
import torch.nn as nn
import torch.nn.functional as F

from _Network.base_module import Network_Opts_base, Network_base,\
                                 disp_to_depth, ResnetEncoder_fromV2,\
                                 DepthDecoder

class Network_Opts(Network_Opts_base):
    def __init__(self):
        super().__init__()
        self.max_depth = 100
        self.min_depth = 0.1
        self.backbone = 18
        self.sample_mode = "nearest"

        self.use_depth = True

class BaseMonoDepthEstNetwork(Network_base):
    """The following parameters should be included in opts:
    min_depth, max_depth: min and max depth in predict
    """
    def __init__(self, opts, *args, **kargs):
        super().__init__(*args, **kargs)
        self.opts = opts

        self.networks["Encoder"] = ResnetEncoder_fromV2(self.opts.backbone,
                                                        True)
        if self.opts.backbone > 34:
            num_ch_enc = [64, 256, 512, 1024, 2048]
        else:
            num_ch_enc = [64, 64, 128, 256, 512]

        self.networks["Decoder"] = DepthDecoder(num_ch_enc,
                                                num_output_channels=1)

        self.all_net = nn.ModuleList(v for k, v in self.networks.items())

        if self.logger.is_train:
            self.logger.make_logdir("Disp")

    def forward(self, inputs, stage):
        image = inputs[("color_aug", "l")]
        features = self.networks["Encoder"](image)
        outputs = self.networks["Decoder"](features)

        if not self.opts.min_depth:
            K = inputs["K"]
            T = inputs["stereo_T"]
            w = image.size()[3]
            self.compute_min_depth(K, T, w)

        # compute depth from disp
        for scale in range(3, -1, -1):
            disp = outputs[("disp", scale)]
            if self.opts.use_depth:
                _, depth = disp_to_depth(disp, self.opts.min_depth,
                                        self.opts.max_depth)
            else:
                fx = inputs["K"][0, 0, 0]
                img_width = image.size()[-1]
                depth = fx/(img_width * 0.3 * disp + 1e-10)
            outputs[("depth", scale)] = depth

        if stage.is_visual and self.logger.is_train:
            self.logger.log_print(self.logger.step)
            v_dict = OrderedDict()
            v_modes = []
            v_size = [["img"]]

            v_dict["img"] = image.detach()
            v_modes.append("img")
            for scale in range(0, 4):
                show_disp = outputs[("disp", scale)].detach()
                self.logger.log_print(show_disp.mean())
                v_dict["disp_{}".format(scale)] = show_disp
                v_modes.append("disp")
                v_size.append(["disp_{}".format(scale)])

            self.logger.do_visualizion("Disp", v_dict, v_modes, v_size, "disp")
            self.logger.log_print("==========")

        return outputs

    def check_info(self):
        assert self.opts.height % 32 == 0, "'height' must be a multiple of 32"
        assert self.opts.width % 32 == 0, "'width' must be a multiple of 32"

    def get_trainable_params(self):
        names = ["Encoder", "Decoder"]
        muls = [1, 1]
        return self.get_module_params(names, muls)

    def compute_min_depth(self, K, T, w):
        fx = K[0, 0, 0]
        baseline = torch.abs(T[0, 0, 3])
        target_d = fx * baseline / (0.3 * w)
        disp_min = 1 / self.opts.max_depth
        disp_max = (1 - disp_min * 0.5 * target_d) / (0.5 * target_d)
        disp_max = disp_max.data
        self.opts.min_depth = 1 / disp_max
        print(self.opts.min_depth)


from collections import OrderedDict

from malcolm.core.loggable import Loggable

class Block(Loggable):
    """Object consisting of a number of Attributes and Methods"""

    def __init__(self, name):
        """
        Args:
            name (str): Block name e.g. "BL18I:ZEBRA1"
        """
        super(Block, self).__init__(logger_name=name)
        self.name = name
        self._methods = OrderedDict()
        self._attributes = OrderedDict()

    def add_attribute(self, attribute):
        """Add an Attribute to the block and set the block as its parent"""

        assert attribute.name not in self._attributes, \
            "Attribute %s already defined for Block %s" \
            % (attribute.name, self.name)
        self._attributes[attribute.name] = attribute
        attribute.set_parent(self)
        setattr(self, attribute.name, attribute)

    def add_method(self, method):
        """Add a Method to the Block

        Args:
            method (Method): The Method object that has already been filled in
        """
        assert method.name not in self._methods, \
            "Method %s already defined for Block %s" % (method.name, self.name)
        self._methods[method.name] = method
        setattr(self, method.name, method)

    def handle_request(self, request):
        """
        Process the request depending on the type

        Args:
            request(Request): Request object specifying action
        """
        self.log_debug("Received request %s", request)
        if request.type_ == request.POST:
            method_name = request.endpoint[-1]
            self._methods[method_name].handle_request(request)
        else:
            layer = self
            for next_link in request.endpoint[1:]:
                layer = getattr(layer, next_link)

            if hasattr(layer, "to_dict"):
                request.respond_with_return(layer.to_dict())
            else:
                request.respond_with_return(layer)

    def to_dict(self):
        """Convert object attributes into a dictionary"""

        d = OrderedDict()

        for attribute_name, attribute in self._attributes.items():
            d[attribute_name] = attribute.to_dict()
        for method_name, method in self._methods.items():
            d[method_name] = method.to_dict()

        return d


"""
Chart objects and methods for displaying data.
"""
#pylint: disable=too-many-arguments,too-many-instance-attributes,protected-access
from collections import OrderedDict
from datetime import datetime, timedelta

from django.template import loader, Context

from dancohenio.utils import avg
from .colors import rgba

class Chart(object):
    """
    Basically a task, but not reflected by a database construct, which
    allows us to duplicate without altering data or making calls to the
    db.
    """
    def __init__(self, task, title=None, width=400, height=200, fill=True):
        """Creates a new chart instance."""
        self.width = width
        self.height = height
        self.fill = fill
        self.task = task
        self.pk = task.pk
        self.uom = task.unit_of_measure

        if title is not None:
            self.title = title
        else:
            self.title = task.name

        self.reference = self.title.replace(' ', '_').lower()
        self.name = task.name
        self.color = task.color
        self.background_color = task.background_color
        self.entries = task.entries
        self.x_data = [e.date for e in self.entries]
        self.y_data = [e.value for e in self.entries]

        self.subcharts = [self]

    def __repr__(self):
        """Python representation of the object."""
        return "<Chart %s - %d Entries>" % (self.title, self.count())

    def __nonzero__(self):
        """Return true if the chart would have at least one point on it."""
        if self.count() > 0:
            return True
        else:
            return False

    def resample(self, by='M', agg=avg):
        """Resample the entries of this chart."""
        sampled_dates, sampled_values = resample(self.entries, by=by, agg=agg)
        self.x_data = sampled_dates
        self.y_data = sampled_values

    def mavg(self, lookback):
        """Recalculates X & Y value as moving averages."""
        x, y = mavg(self.entries, lookback=lookback)
        self.x_data = x
        self.y_data = y

    def _dataset_js(self):
        """JS representation of the main dataset of this chart."""
        data = {
            'name': self.name,
            'fill': bool_js(self.fill),
            'color': self.color,
            'bg_color': self.background_color,
            'line_tension': 0,
            'border_width': .5,
            'point_border_width': 1,
            'point_hover_radius': 5,
            'point_hover_border_width': 1,
            'point_radius': 1,
            'point_hit_radius': 10,
            'y': self._y()
        }
        js = """{
        label: "%(name)s",
        fill: %(fill)s,
        borderColor: "%(color)s",
        backgroundColor: "%(bg_color)s",
        lineTension: %(line_tension)s,
        borderWith: %(border_width)s,
        pointBorderWidth: %(point_border_width)s,
        pointHoverRadius: %(point_hover_radius)s,
        pointHoverBorderWidth: %(point_hover_border_width)s,
        pointRadius: %(point_radius)s,
        pointHitRadius: %(point_hit_radius)s,
        data: [%(y)s]
        }""" % data
        return js

    def _datasets(self):
        """Return all datasets for this chart as js formatted string."""
        return ','.join([ch._dataset_js() for ch in self.subcharts])

    def _render_datasets(self):
        """Set context values for the subhcarts."""
        datasets = []
        for ch in self.subcharts:
            d = {
                'name': ch.name,
                'fill': bool_js(ch.fill),
                'color': ch.color,
                'bg_color': ch.background_color,
                'line_tension': 0,
                'border_width': .5,
                'point_border_width': 1,
                'point_hover_radius': 5,
                'point_hover_border_width': 1,
                'point_radius': 1,
                'point_hit_radius': 10,
                'y': ch._y()
            }
            datasets.append(d)
        return datasets

    def render(self):
        """Render the chart as js."""
        context = {
            'reference': self.reference,
            'labels': self._x(),
            'datasets': self._render_datasets(),
        }
        template = loader.get_template('focus/chart_template.js')
        return template.render(Context(context))

    def count(self):
        """Returns count of entries."""
        assert len(self.x_data) == len(self.y_data)
        return len(self.x_data)

    def _x(self):
        """
        Return the x axis formatted as needed.

        fmt: Dec. 19, 2012
        """
        dates = []
        for x in self.x_data:
            try:
                date = x.date().strftime('%b. %d, %Y')
            except AttributeError:
                date = x
            dates.append(date)
        return ','.join(['"%s"' % x for x in dates])

    def _y(self):
        """
        Returns a nicely formatted y axis.
        """
        return ','.join([str(y) for y in self.y_data])

def bool_js(boolean):
    """Return a python boolean as js boolean text."""
    if boolean:
        return "true"
    else:
        return "false"

class ResampledChart(Chart):
    """Creates a Chart object that has resampled its data."""
    def __init__(self, task, by='M', agg=avg, title=None, width=400, height=200, fill=True):
        """Instantiates the object."""
        super(ResampledChart, self).__init__(task, title=title, width=width,
                                             height=height, fill=fill)
        resampled_dates, resampled_values = resample(self.entries, by=by, agg=agg)
        self.x_data = resampled_dates
        self.y_data = resampled_values

class MovingAverageChart(Chart):
    """Creates a chart with a calculated moving average."""
    def __init__(self, task, lookback, title=None, fill=False):
        """
        Instantiates the object and calculates a moving average for the data.
        """
        super(MovingAverageChart, self).__init__(task, title, fill=fill)
        x, y = mavg(self.entries, lookback=lookback)
        self.x_data = x
        self.y_data = y

class DualMovingAverageChart(MovingAverageChart):
    """
    Graphs the task with two dual moving averages.
    """
    def __init__(self, task, short_val, long_val, title="Dual Moving Averages"):
        """
        Creates several chart instances and joins them as subcharts.
        """
        super(DualMovingAverageChart, self).__init__(
            task, lookback=short_val, title=title, fill=False
            )
        self.subcharts[0].name = 'Short'
        long_chart = MovingAverageChart(task, lookback=long_val)
        long_chart.name = 'Long'
        long_chart.color = rgba(task.red, task.green, task.blue, .4)

        # equalize lengths
        max_length = min([len(self.y_data), len(long_chart.y_data)])
        self.x_data = self.x_data[-max_length:]
        self.y_data = self.y_data[-max_length:]
        long_chart.x_data = long_chart.x_data[-max_length:]
        long_chart.y_data = long_chart.y_data[-max_length:]
        #assert self.count() == long_chart.count()

        self.subcharts.append(long_chart)

class MovingAverageEnvelopeChart(MovingAverageChart):
    """
    Graphs the task with two or more moving averages.
    """
    def __init__(self, task, bands, title="Moving Average Envelope", fill=False):
        """
        Takes a task and a list of lookback values and creates a chart of
        multiple bands of moving averages.
        """
        super(MovingAverageEnvelopeChart, self).__init__(
            task, lookback=bands[0], title=title, fill=fill
            )
        self.subcharts[0].name = bands[0]
        for i in bands[1:]:
            subchart = MovingAverageChart(task, lookback=i, fill=fill)
            subchart.name = i
            subchart.color = rgba(task.red, task.green, task.blue, 1 - .1*i)
            self.subcharts.append(subchart)

        # equalize lengths
        max_length = min([len(ch.x_data) for ch in self.subcharts])

        self.x_data = self.x_data[-max_length:]
        self.y_data = self.y_data[-max_length:]

        for ch in self.subcharts:
            ch.x_data = ch.x_data[-max_length:]
            ch.y_data = ch.y_data[-max_length:]
            #assert self.count() == long_chart.count()

class TimeFrameChart(Chart):
    """Chart that is started/stopped at given times."""
    def __init__(self, task, fromdate, todate, title):
        super(TimeFrameChart, self).__init__(task, title=title)
        selected = [e for e in self.entries
                    if e.date >= fromdate and e.date <= todate]
        self.entries = selected
        self.x_data = [e.date for e in selected]
        self.y_data = [e.value for e in selected]

def mavg(entries, lookback):
    """
    Returns two lists, one for each of x_axis and y_axis where the
    y axis is a calculated moving average of the given count.
    """
    logged, x, y = [], [], []
    for e in entries:
        logged.append(e)
        if len(logged) < lookback:
            continue
        x.append(e.date)
        sample = [float(e.value) for e in logged[-lookback:]]
        assert len(sample) == lookback
        y.append(avg(sample))
    return x, y

def resample(entries, by='M', agg=avg):
    """
    Resample date entries by a group of size [default: M]
    """
    groups = OrderedDict()
    for e in entries:
        if by == 'M':
            sample = reduce_by_month(e.date)
        elif by == 'W':
            sample = reduce_by_week(e.date)
        elif by == 'Y':
            sample = e.date.year
        elif by == 'd':
            sample = reduce_by_day(e.date)
        else:
            raise NotImplementedError

        val = float(e.value)

        try:
            groups[sample].append(val)
        except KeyError:
            groups[sample] = [val]

    groups = ffill(groups, by=by)

    x, y = [], []
    for group, values in groups.items():
        x.append(group)
        y.append(agg(values))

    return x, y

def reduce_by_month(date):
    """Returns a datetime object that is the first of its month."""
    return datetime(date.year, date.month, 1, tzinfo=date.tzinfo)

def reduce_by_week(date):
    """Returns a datetime object representing the first day of that week."""
    date = reduce_by_day(date)
    return date - timedelta(days=date.isocalendar()[2] - 1)

def reduce_by_day(dateish):
    """Turn a datetime object into a date, leave dates alone."""
    try:
        return dateish.date()
    except AttributeError:
        return dateish

def advance_by_month(date):
    """Returns the date one month later."""
    if date.month == 12:
        return date.replace(year=date.year + 1, month=1)
    else:
        return date.replace(month=date.month + 1)

def ffill(groups, by='M', fill_value=0., fill_method=None):
    """
    Takes a group dict and fills missing time keys with the value in
    fill_value.
    """
    if not groups: return groups

    if by == 'd':
        dates = [reduce_by_day(d) for d in groups.keys()]
        dates.sort()
        step = lambda x: x + timedelta(days=1)
    elif by == 'M':
        dates = [reduce_by_month(d) for d in groups.keys()]
        dates.sort()
        step = advance_by_month
    elif by == 'W':
        dates = [reduce_by_week(d) for d in groups.keys()]
        dates.sort()
        step = lambda x: x + timedelta(days=7)
    elif by == 'Y':
        dates = [d for d in groups.keys()]
        dates.sort()
        step = lambda x: x + 1
    else:
        raise NotImplementedError

    d = dates[0]
    previous_value = fill_value
    while d <= dates[-1]:
        if d not in groups.keys():
            if fill_method == 'previous':
                groups[d] = previous_value
            else:
                groups[d] = [fill_value]
        else:
            previous_value = groups[d]
        d = step(d)

    sorted_groups = OrderedDict()
    for key in sorted(groups.keys()):
        sorted_groups[key] = groups[key]
    return sorted_groups

def filter_dates(entries, fromdate, todate):
    """
    Filters entries by input dates
    """
    return [e for e in entries if e.date >= fromdate and e.date <= todate]

def x_data(dates):
    """
    Return the x axis formatted as needed.

    fmt: Dec. 19, 2012
    """
    date_strings = []
    for x in dates:
        try:
            date = x.date().strftime('%b. %d, %Y')
        except AttributeError:
            date = x
        date_strings.append(date)
    return ','.join(['"%s"' % x for x in date_strings])

def y_data(values):
    """
    Returns a nicely formatted y axis.
    """
    return ','.join([str(y) for y in values])

def equalize_lengths(axes):
    """Equalize the lengths of all the input axes."""
    max_length = min([len(axis[0]) for axis in axes])
    print('Maximum length should be set to %d' % max_length)
    new_axes = []
    for x, y in axes:
        x = x[-max_length:]
        y = y[-max_length:]
        print(len(x), len(y))
        new_axes.append([x, y])
    return new_axes


from __future__ import division

from math import ceil
from time import time
from itertools import groupby
from collections import deque

import six
from numpy import arange, mean
import matplotlib.pyplot as plt

from neupy.utils import format_data, is_row1d
from neupy.helpers import preformat_value
from neupy.core.base import BaseSkeleton
from neupy.core.properties import (Property, FuncProperty, NumberProperty,
                                   BoolProperty)
from neupy.layers import BaseLayer, OutputLayer
from neupy.functions import normilize_error_output, mse
from .utils import iter_until_converge, shuffle
from .connections import (FAKE_CONNECTION, LayerConnection,
                          NetworkConnectionError)

__all__ = ('BaseNetwork',)

def show_training_summary(network):
    network.logs.data("""
        Epoch {epoch}
        Train error:  {error}
        Validation error: {error_out}
        Epoch time: {epoch_time} sec
    """.format(
        epoch=network.epoch,
        error=network.last_error_in() or '-',
        error_out=network.last_error_out() or '-',
        epoch_time=round(network.train_epoch_time, 5)
    ))

def show_epoch_summary(network, show_epoch):
    delay_limit = 1  # in seconds
    prev_summary_time = None
    delay_history_length = 10
    terminal_output_delays = deque(maxlen=delay_history_length)

    while True:
        now = time()

        if prev_summary_time is not None:
            time_delta = now - prev_summary_time
            terminal_output_delays.append(time_delta)

        show_training_summary(network)
        prev_summary_time = now

        if len(terminal_output_delays) == delay_history_length:
            prev_summary_time = None
            average_delay = mean(terminal_output_delays)

            if average_delay < delay_limit:
                show_epoch *= ceil(delay_limit / average_delay)
                network.logs.warning(
                    "Too many outputs in a terminal. Set "
                    "up logging after each {} epoch"
                    "".format(show_epoch)
                )
                terminal_output_delays.clear()

        yield show_epoch

def shuffle_train_data(input_train, target_train):
    if target_train is None:
        return shuffle(input_train), None
    return shuffle(input_train, target_train)

def clean_layers(connection):
    """ Clean layers connections and format transform them into one format.
    Also this function validate layers connections.

    Parameters
    ----------
    connection : list, tuple or object
        Layers connetion in different formats.

    Returns
    -------
    object
        Cleaned layers connection.
    """
    if connection == FAKE_CONNECTION:
        return connection

    if isinstance(connection, tuple):
        connection = list(connection)

    islist = isinstance(connection, list)

    if islist and isinstance(connection[0], BaseLayer):
        chain_connection = connection.pop()
        for layer in reversed(connection):
            chain_connection = LayerConnection(layer, chain_connection)
        connection = chain_connection

    elif islist and isinstance(connection[0], LayerConnection):
        pass

    if not isinstance(connection.output_layer, OutputLayer):
        raise NetworkConnectionError("Final layer must be OutputLayer class "
                                     "instance.")

    return connection

def parse_show_epoch_property(value, n_epochs):
    if isinstance(value, int):
        return value

    number_end_position = value.index('time')
    # Ignore grammar mistakes like `2 time`, this error could be
    # really annoying
    n_epochs_to_check = int(value[:number_end_position].strip())

    if n_epochs <= n_epochs_to_check:
        return 1

    return int(round(n_epochs / n_epochs_to_check))

class ShowEpochProperty(Property):
    """ Class helps validate specific syntax for `show_epoch`
    property from ``BaseNetwork`` class.
    """
    expected_type = tuple([int] + [six.string_types])

    def validate(self, value):
        if not isinstance(value, six.string_types):
            if value < 1:
                raise ValueError("Property `{}` value should be integer "
                                 "greater than zero or string. See the "
                                 "documentation for more information."
                                 "".format(self.name))
            return

        if 'time' not in value:
            raise ValueError("`{}` value has invalid string format."
                             "".format(self.name))

        valid_endings = ('times', 'time')
        number_end_position = value.index('time')
        number_part = value[:number_end_position].strip()

        if not value.endswith(valid_endings) or not number_part.isdigit():
            valid_endings_formated = ', '.join(valid_endings)
            raise ValueError(
                "Property `{}` in string format should be a positive number "
                "with one of those endings: {}. For example: `10 times`."
                "".format(self.name, valid_endings_formated)
            )

        if int(number_part) < 1:
            raise ValueError("Part that related to the number in `{}` "
                             "property should be an integer greater or "
                             "equal to one.".format(self.name))

class BaseNetwork(BaseSkeleton):
    """ Base class Network algorithms.

    Parameters
    ----------
    {full_params}

    Methods
    -------
    {plot_errors}
    {last_error}
    """
    error = FuncProperty(default=mse)
    use_bias = BoolProperty(default=True)
    step = NumberProperty(default=0.1)

    # Training settings
    show_epoch = ShowEpochProperty(min_size=1, default='10 times')
    shuffle_data = BoolProperty(default=False)

    # Signals
    train_epoch_end_signal = FuncProperty()
    train_end_signal = FuncProperty()

    def __init__(self, connection, **options):
        self.connection = clean_layers(connection)

        self.errors_in = []
        self.errors_out = []
        self.train_epoch_time = None

        self.layers = list(self.connection)
        self.input_layer = self.layers[0]
        self.output_layer = self.layers[-1]
        self.train_layers = self.layers[:-1]

        super(BaseNetwork, self).__init__(**options)
        self.setup_defaults()

        if self.verbose:
            self.show_network_options(highlight_options=options)

        self.init_layers()

    def show_network_options(self, highlight_options=None):
        available_classes = [cls.__name__ for cls in self.__class__.__mro__]
        logs = self.logs

        if highlight_options is None:
            highlight_options = {}

        def classname_grouper(option):
            classname = option[1].class_name
            class_priority = -available_classes.index(classname)
            return (class_priority, classname)

        # Sort and group options by classes
        grouped_options = groupby(
            sorted(self.options.items(), key=classname_grouper),
            key=classname_grouper
        )

        if isinstance(self.connection, LayerConnection):
            logs.header("Network structure")
            logs.log("LAYERS", self.connection)

        # Just display in terminal all network options.
        logs.header("Network options")
        for (_, clsname), class_options in grouped_options:
            if not class_options:
                # When in some class we remove all available attributes
                # we just skip it.
                continue

            logs.simple("{}:".format(clsname))

            for key, data in sorted(class_options):
                if key in highlight_options:
                    logger = logs.log
                    value = highlight_options[key]
                else:
                    logger = logs.gray_log
                    value = data.value

                logger("OPTION", "{} = {}".format(
                    key, preformat_value(value))
                )
            logs.empty()

    def setup_defaults(self):
        """ Setup default values before populate the options.
        """

    # ----------------- Neural Network Layers ---------------- #

    def init_layers(self):
        """ Initialize layers.
        """
        if self.connection == FAKE_CONNECTION:
            return

        for layer in self.train_layers:
            layer.initialize(with_bias=self.use_bias)

    # ----------------- Neural Network Train ---------------- #

    def _train(self, input_train, target_train=None, input_test=None,
               target_test=None, epochs=100, epsilon=None):
        """ Main method for the Neural Network training.
        """

        # ----------- Pre-format target data ----------- #

        input_row1d = is_row1d(self.input_layer)
        input_train = format_data(input_train, row1d=input_row1d)

        target_row1d = is_row1d(self.output_layer)
        target_train = format_data(target_train, row1d=target_row1d)

        if input_test is not None:
            input_test = format_data(input_test, row1d=input_row1d)

        if target_test is not None:
            target_test = format_data(target_test, row1d=target_row1d)

        # ----------- Validate input values ----------- #

        if epsilon is not None and epochs <= 2:
            raise ValueError("Network should train at teast 3 epochs before "
                             "check the difference between errors")

        # ----------- Predefine parameters ----------- #

        self.epoch = 1
        show_epoch = self.show_epoch
        logs = self.logs
        compute_error_out = (input_test is not None and
                             target_test is not None)
        predict = self.predict
        last_epoch_shown = 0

        if epsilon is not None:
            iterepochs = iter_until_converge(self, epsilon, max_epochs=epochs)

            if isinstance(show_epoch, six.string_types):
                show_epoch = 100
                logs.warning("Can't use `show_epoch` value in converging "
                             "mode. Set up 100 to `show_epoch` property "
                             "by default.")

        else:
            iterepochs = range(1, epochs + 1)
            show_epoch = parse_show_epoch_property(show_epoch, epochs)

        epoch_summary = show_epoch_summary(self, show_epoch)

        # ----------- Train process ----------- #

        logs.header("Start train")
        logs.log("TRAIN", "Train data size: {}".format(input_train.shape[0]))

        if input_test is not None:
            logs.log("TRAIN", "Validation data size: {}"
                              "".format(input_test.shape[0]))

        if epsilon is None:
            logs.log("TRAIN", "Total epochs: {}".format(epochs))
        else:
            logs.log("TRAIN", "Max epochs: {}".format(epochs))

        logs.empty()

        # Optimizations for long loops
        errors = self.errors_in
        errors_out = self.errors_out
        shuffle_data = self.shuffle_data

        error_func = self.error
        train_epoch = self.train_epoch
        train_epoch_end_signal = self.train_epoch_end_signal
        train_end_signal = self.train_end_signal

        self.input_train = input_train
        self.target_train = target_train

        for epoch in iterepochs:
            self.epoch = epoch
            epoch_start_time = time()

            if shuffle_data:
                input_train, target_train = shuffle_train_data(input_train,
                                                               target_train)
                self.input_train = input_train
                self.target_train = target_train

            try:
                error = train_epoch(input_train, target_train)

                if compute_error_out:
                    predicted_test = predict(input_test)
                    error_out = error_func(predicted_test, target_test)
                    errors_out.append(error_out)

                errors.append(error)
                self.train_epoch_time = time() - epoch_start_time

                if epoch % show_epoch == 0 or epoch == 1:
                    show_epoch = next(epoch_summary)
                    last_epoch_shown = epoch

                if train_epoch_end_signal is not None:
                    train_epoch_end_signal(self)

            except StopIteration as err:
                logs.log("TRAIN", "Epoch #{} stopped. {}"
                                  "".format(epoch, str(err)))
                break

        if epoch != last_epoch_shown:
            show_training_summary(self)

        if train_end_signal is not None:
            train_end_signal(self)

        logs.log("TRAIN", "End train")

    # ----------------- Errors ----------------- #

    def _last_error(self, errors):
        if errors and errors[-1] is not None:
            return normilize_error_output(errors[-1])

    def last_error_in(self):
        return self._last_error(self.errors_in)

    def last_error(self):
        return self._last_error(self.errors_in)

    def last_error_out(self):
        return self._last_error(self.errors_out)

    def previous_error(self):
        errors = self.errors_in
        return normilize_error_output(errors[-2]) if len(errors) > 2 else None

    def _normalized_errors(self, errors):
        if not len(errors) or isinstance(errors[0], float):
            return errors

        self.logs.warning("Your errors bad formated for plot output. "
                          "They will be normilized.")

        normilized_errors = []
        for error in errors:
            normilized_errors.append(normilize_error_output(error))

        return normilized_errors

    def normalized_errors_in(self):
        return self._normalized_errors(self.errors_in)

    def normalized_errors_out(self):
        return self._normalized_errors(self.errors_out)

    def plot_errors(self, logx=False):
        if not self.errors_in:
            return

        errors_in = self.normalized_errors_in()
        errors_out = self.normalized_errors_out()
        errors_range = arange(len(errors_in))
        plot_function = plt.semilogx if logx else plt.plot

        line_error_in, = plot_function(errors_range, errors_in)
        title_text = 'Learning error after each epoch'

        if errors_out:
            line_error_out, = plot_function(errors_range, errors_out)
            plt.legend(
                [line_error_in, line_error_out],
                ['Train error', 'Validation error']
            )
            title_text = 'Learning errors after each epoch'

        plt.title(title_text)
        plt.xlim(0)

        plt.ylabel('Error')
        plt.xlabel('Epoch')

        plt.show()

    # ----------------- Representations ----------------- #

    def get_class_name(self):
        return self.__class__.__name__

    def __repr__(self):
        classname = self.get_class_name()
        options_repr = self._repr_options()

        if self.connection != FAKE_CONNECTION:
            return "{}({}, {})".format(classname, self.connection,
                                       options_repr)
        return "{}({})".format(classname, options_repr)


#######################################################
#
# TAKFreeServer.py
# Original author: naman108
# This code is Open Source, made available under the EPL 2.0 license.
# https://www.eclipse.org/legal/eplfaq.php
# credit to Harshini73 for base code
#
#######################################################
import argparse
import datetime
import logging
import os
import socket
import sqlite3
import sys
import threading
import time
import traceback
import uuid
import xml.etree.ElementTree as ET
from logging.handlers import RotatingFileHandler

import constants
import SQLcommands
from Controllers.RequestCOTController import RequestCOTController
from Controllers.serializer import Serializer

const = constants.vars()
sql = SQLcommands.sql()

def newHandler(filename, log_level, log_format):
    handler = RotatingFileHandler(
        filename,
        maxBytes=const.MAXFILESIZE,
        backupCount=const.BACKUPCOUNT
    )
    handler.setFormatter(log_format)
    handler.setLevel(log_level)
    return handler

log_format = logging.Formatter(const.LOGFORMAT)
logger = logging.getLogger(const.LOGNAME)
logger.setLevel(logging.DEBUG)
logger.addHandler(newHandler(const.DEBUGLOG, logging.DEBUG, log_format))
logger.addHandler(newHandler(const.WARNINGLOG, logging.WARNING, log_format))
logger.addHandler(newHandler(const.INFOLOG, logging.INFO, log_format))
console = logging.StreamHandler(sys.stdout)
console.setFormatter(log_format)
console.setLevel(logging.DEBUG)
logger.addHandler(console)

''' Server class '''
class ThreadedServer(object):
    def __init__(self, host=const.IP, port=const.PORT):
        # change from string
        self.host = host
        self.port = port
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.sock.bind((self.host, self.port))
        self.client_dict = {}
        logger.info(f"Server IP: {host}, server port: {port}")
        self.emergencyDict = {}
        # configure sql database
        with sqlite3.connect(const.DATABASE) as db:
            cursor = db.cursor()
            cursor.execute(sql.CREATEUSERSTABLE)
            cursor.close()
            db.commit()
        self.bandaidUID = ''

    def listen(self):
        '''
        listen for client connections and begin thread if found
        '''
        threading.Thread(target=self.bandaid, args=(), daemon=True).start()
        self.sock.listen(1000)
        while True:
            try:
                client, address = self.sock.accept()
                threading.Thread(target=self.listenToClient, args=(client, address), daemon=True).start()
            except:
                logger.error(traceback.format_exc())
                logger.error('Error in listen()')

    def bandaid(self):
        while True:
            try:
                start = datetime.datetime.now()
                end = start + datetime.timedelta(minutes=const.RENEWTIME)
                while datetime.datetime.now() < end:
                    time.sleep(10)
                self.bandaidUID = uuid.uuid1()
                mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                mysock.connect(('127.0.0.1', const.PORT))
                mysock.send(Serializer().serializerRoot(RequestCOTController().ping(eventuid=self.bandaidUID)).encode())
                mysock.recv(2048)
                mysock.shutdown(socket.SHUT_RDWR)
                mysock.close()
                logger.info('finished bandaid keepalive')
                logger.debug(f"Currently running {len(threading.enumerate())} threads")
            except ConnectionRefusedError:
                logger.warning("Bandaid listening socket was closed")
            except:
                logger.error(traceback.format_exc())
                logger.error("Error in bandaid()")

    def check_xml(self, xml_string, current_id):
        '''
        check xml type or class
        '''
        data_value = ''
        try:
            if xml_string == const.EMPTY_BYTE:
                logger.info('client disconnected via empty byte response')
                self.client_dict[current_id]['alive'] = 0
                logger.info(str(self.client_dict[current_id]['uid'])+' disconnected')
                return const.FAIL
            elif xml_string == None:
                logger.info('client disconnected via none response')
                self.client_dict[current_id]['alive'] = 0
                logger.info(str(self.client_dict[current_id]['uid'])+' disconnected')
                return const.FAIL

            tree = ET.fromstring(xml_string)
            uid = tree.get('uid')
            logger.debug('parsing data uid is ' + str(uid))
            cot_type = tree.get('type')
            if cot_type == "a-f-G-U-C":
                self.client_dict[current_id]['id_data'] = xml_string
            elif cot_type == 'b-f-t-a':
                destination = tree.find('detail').find('marti').find('dest').attrib['callsign']
                connData = self.client_dict[current_id]["id_data"]
                for x in self.client_dict:
                    if self.client_dict[x]["callsign"] == destination:
                        self.client_dict[x]["main_data"].append(connData)
                        logger.info('adding conn data to '+str(x))
                        logger.info(f"Now adding the following connection data: {str(connData)}")
            if uid.endswith("9-1-1"):
                for x in tree.iter('emergency'):
                    if x.get('cancel') != 'true':
                        self.emergencyDict[uid] = xml_string
                    else:
                        del self.emergencyDict[uid]
            elif uid.endswith(const.PING):
                data_value = const.PING
                logger.debug(f"Received a ping: {xml_string}")
            elif uid.startswith(const.GEOCHAT):
                data_value = const.GEOCHAT
                logger.debug(f"Received a GeoChat: {xml_string}")
            else:
                logger.debug(f"Received CoT: {xml_string}")

            # adds data to all connected client data list except sending client
            for detail in tree.findall('detail'):
                marti = detail.find('marti')
                if marti != None:
                    dest = marti.find('dest')
                    callsign = dest.attrib['callsign']
                    for client_id in self.client_dict:
                        if self.client_dict[client_id]['callsign'] == callsign:
                            self.client_dict[client_id]['main_data'].append(xml_string)
                else:
                    for client_id in self.client_dict:
                        if client_id != current_id:
                            self.client_dict[client_id]['main_data'].append(xml_string)
            return data_value

        except:
            logger.error(traceback.format_exc())
            logger.error(f"Error in check_xml for: {xml_string}")

    def connectionSetup(self, client, address):
        db = sqlite3.connect(const.DATABASE)
        try:
            cursor = db.cursor()
            first_run = 1
            # Create client dictionary within main dictionary containing arrays for data and chat also other stuff for client initial connection
            total_clients_connected = 0
            total_clients_connected += 1
            id_data = client.recv(const.STARTBUFFER)
            logger.debug(f"id_data = {id_data}")
            tree = ET.fromstring(id_data)
            uid = tree.get('uid')
            if uid == self.bandaidUID:
                return 'Bandaid'
            callsign = tree.find('detail').find('contact').attrib['callsign']
            current_id = uuid.uuid1().int

            # add identifying information
            self.client_dict[current_id] = {
                'id_data': id_data,
                'main_data': [],
                'alive': 1,
                'uid': uid,
                'client': client,
                'callsign': callsign
            }
            cursor.execute(sql.INSERTNEWUSER, (str(current_id), str(uid), str(callsign)))
            cursor.close()
            db.commit()
            logger.info(f"Client connected, initial information for current_id={current_id}: {self.client_dict[current_id]}")
            return str(first_run)+' ? '+str(total_clients_connected)+' ? '+str(id_data)+' ? '+str(current_id)
        except:
            logger.error(traceback.format_exc())
            logger.error('Error in connection setup')
            return "error"
        finally:
            db.close()

    def recieveAll(self, client):
        try:
            total_data = []
            while True:
                data = client.recv(const.BUFFER)
                logger.debug(f"Received {sys.getsizeof(data)} bytes from {client}")
                if sys.getsizeof(data) == const.BUFFER+33:
                    total_data.append(data)
                elif sys.getsizeof(data) < const.BUFFER+33:
                    total_data.append(data)
                    break
            total_data = b''.join(total_data)
            return total_data
        except:
            logger.error(traceback.format_exc())
            logger.error(f"Error in recieveAll() from {client}")
            return None

    def listenToClient(self, client, address):
        '''
        Function to receive data from the client. this must be long as everything
        '''
        try:
            defaults = self.connectionSetup(client, address)
            if defaults == 'error':
                client.shutdown(socket.SHUT_RDWR)
                client.close()
                return
            elif defaults == 'Bandaid':
                self.sock.shutdown(socket.SHUT_RDWR)
                client.close()
                return
            else:
                defaults = defaults.split(' ? ')
                logger.debug(defaults)
                first_run = int(defaults[0])
                id_data = bytes(defaults[2], 'utf-8')
                current_id = int(defaults[3])
                # main connection loop
                while True:
                    # Receive data
                    try:
                        if first_run == 0:
                            data = self.recieveAll(client)
                            logger.debug(f"Received data from client: {str(data)}")
                            working = self.check_xml(data, current_id)
                            # checking if check_xml detected client disconnect
                            if working == const.FAIL:
                                timeoutInfo = Serializer().serializerRoot(RequestCOTController().timeout(
                                    eventhow='h-g-i-g-o',
                                    eventuid=uuid.uuid1(),
                                    linkuid=self.client_dict[current_id]['uid']
                                ))
                                logger.debug(f"Sending timeout: {timeoutInfo.encode()}")
                                for client_id in self.client_dict:
                                    if client_id != current_id:
                                        self.client_dict[client_id]['client'].send(timeoutInfo.encode())
                                uid = self.client_dict[current_id]['uid']
                                del self.client_dict[current_id]
                                with sqlite3.connect(const.DATABASE) as db:
                                    cursor = db.cursor()
                                    cursor.execute(sql.DELETEBYUID, (uid,))
                                    cursor.close()
                                    db.commit()
                                    client.shutdown(socket.SHUT_RDWR)
                                    client.close()
                                return
                            elif working == const.PING:
                                logger.debug('Received ping')

                        elif first_run == 1:
                            for client_id in self.client_dict:
                                client = self.client_dict[client_id]['client']
                                if client != self.client_dict[current_id]['client']:
                                    logger.info('Sending '+str(id_data))
                                    client.send(self.client_dict[current_id]['id_data'])
                            for client_id in self.client_dict:
                                data = self.client_dict[client_id]['id_data']
                                logger.debug('Sending conn data to '+str(client))
                                client.send(data)
                            threading.Thread(
                                target=self.sendClientData,
                                args=(client, address, current_id),
                                daemon=True).start()

                        # just some debug stuff
                        first_run = 0
                    except:
                        logger.error(traceback.format_exc())
                        logger.error('Error in listenToClient() main loop')
                        client.close()
                        return
        except Exception as e:
            logger.error(traceback.format_exc())
            logging.error("Unknown error in listenToClient")
            client.close()

    def sendClientData(self, client, address, current_id):
        try:
            while True:
                time.sleep(const.DELAY)
                for uid in self.emergencyDict:
                    client.send(self.emergencyDict[uid])
                    logger.info(f"Emergency activated: {uid}")

                if len(self.client_dict[current_id]['main_data']) > 0:
                    for x in self.client_dict[current_id]['main_data']:
                        logger.debug(self.client_dict[current_id]['main_data'])
                        client.send(x)
                        logger.info('Sent ' + str(x) + ' to ' + str(address))
                        self.client_dict[current_id]['main_data'].remove(x)
                else:
                    client.send(Serializer().serializerRoot(RequestCOTController().ping(eventuid=uuid.uuid1())).encode())
        except:
            logger.error(traceback.format_exc())
            logger.warning('Error in sendClientData')
        finally:
            client.close()

def startup():
    logger.info('starting windows service')
    ThreadedServer(host=const.IP, port=const.PORT).listen()

if __name__ == "__main__":
    try:
        parser = argparse.ArgumentParser()
        parser.add_argument("-p", type=int)
        args = parser.parse_args()
        port = args.p
    except:
        ThreadedServer(host='', port=const.PORT).listen()
        logger.error(f"Failed to read port number from command arguments, defaulting to {const.PORT}")
        port = const.PORT
    ThreadedServer(host=const.IP, port=port).listen()


from collections import Counter
from typing import List

class BingoCard:
    """Represents a bingo card."""

    WIDTH = 5
    MARKED = -1
    MARKED_WIN = MARKED * WIDTH

    def __init__(self, lines: List[str]) -> None:
        """Initialize the bingo card with its lines.

        Args:
            lines (List[str]): lines of the bingo card

        """
        self.rows: List[List[int]] = []
        for line in lines:
            self.rows.append([int(num) for num in line.split()])

    def mark(self, marker: int) -> None:
        """Mark the bingo board with a marker number.

        Args:
            marker (int): the called number

        """
        self.rows = [
            [num if num != marker else self.MARKED for num in row]
            for row in self.rows
            ]

    def is_winning(self, numbers: List[int]) -> bool:
        """Determine if the sequence of numbers is a win.

        Returns:
            bool: the sequence of numbers is summed and only contains marked
                numbers

        """
        count = Counter(numbers)
        return len(count) == 1 and sum(numbers) == self.MARKED_WIN

    def has_won(self) -> bool:
        """Check if the bingo card has won.

        Returns:
            bool: True if the board is a winner; False otherwise

        """
        won_row = any([self.is_winning(row) for row in self.rows])
        cols = [
            [row[n] for row in self.rows]
            for n, _ in enumerate(self.rows[0])
            ]
        won_col = any([self.is_winning(col) for col in cols])

        # RIP my reading comprehension
        # diag_l = []
        # diag_r = []
        # for n in range(self.WIDTH):
        #     diag_l.append(self.rows[n][n])
        #     diag_r.append(self.rows[n][self.WIDTH - n - 1])
        # won_diag = any(
        #     [self.is_winning(diag_l), self.is_winning(diag_r)]
        #     )

        return any([won_row, won_col])

    def sum_all_unmarked(self) -> int:
        """Sum all unmarked bingo numbers.

        Returns:
            int: sum of all unmarked numbers

        """
        rows = [[n for n in row if n != self.MARKED] for row in self.rows]
        return sum([sum(row) for row in rows])


import numpy as np
import math
import matplotlib.pyplot as plt

def target(w, x, y): #target function of log likehood method
    ret = 0
    n = len(x)
    for i in range(0,n):
        #print(x[i], w)
        rate = w.dot(x[i])
        #print(w, x[i], rate)
        ret += -rate*y[i] + math.log(1+math.exp(rate))
    return ret
pass
def test(w, testx, testy): #test and return accuracy
    n = len(testx)
    sum = 0
    for i in range(0,n):
        rate = w.dot(testx[i])
        #print("here",w, testx[i], rate)
        activation = 1./(math.exp(-rate)+1.)
        #print (rate)
        #print ("act", activation)
        if activation < 0.5 and testy[i] == 0 or (activation > 0.5 and testy[i] == 1):
           sum += 1
        elif activation == 0.5:
            sum += 0.5
    return sum/n
pass
def newton(weight, trainx, trainy):#Newton optimization of log likehood
    cur = 0
    n = len(trainx[0])
    while 1:
        old = cur
        #print(weight, trainx, trainy)
        cur = target(weight, trainx, trainy)
        print(cur, old)
        if abs(old-cur) <= 0.000001:
            break
        dl = np.matrix(np.zeros(n)).astype(float)
        ddl = np.zeros((n,n)).astype(float)
        #print (n,dl, ddl)
        for i in range(0,len(trainx)):
            rate = weight.dot(trainx[i])
            #print(weight, trainx[i], weight.dot(trainx[i]))
            p = 1. - 1./(1.+math.exp(rate))
            #print("p=", p)
            #print(dl)
            dl += (-trainy[i]+p)*trainx[i]
            #print (i, np.matrix(trainx[i]).T, trainx[i], np.matrix(trainx[i]).T*trainx[i])
            ddl += np.matrix(trainx[i]).T*trainx[i]*p*(1-p)
        #print ("dl,ddl=",dl, np.linalg.pinv(ddl),dl*np.linalg.pinv(ddl))
        weight -= dl*np.linalg.pinv(ddl)
        #print(cur, weight)
    #print(test(weight, testx, testy))
    return weight
pass
def kFoldeCrossValidation(k, data, y):
    n = len(data[0])
    weight_final = np.zeros(n).astype(float)
    weight_final = np.matrix(weight_final)
    for it in range(0,k):
        weight = np.zeros(n).astype(float)#initialize train data and test data
        weight = np.matrix(weight)
        trainx = data[:math.floor(n*it/k)] + data[math.floor(n*(it+1)/k):]
        trainy = y[:math.floor(n*it/k)] + y[math.floor(n*(it+1)/k):]
        testx = data[math.floor(n*it/k):math.floor(n*(it+1)/k)]
        testy = y[math.floor(n*it/k):math.floor(n*(it+1)/k)]

        testx = np.array(testx).astype(float)#transfer data into array structure
        testy = np.array(testy).astype(int)
        trainx = np.array(trainx).astype(float)
        trainy = np.array(trainy).astype(int)
        #print(trainx, trainy)
        weight = newton(weight, trainx, trainy)

        weight_final += weight
    weight_final /= k
    data = np.array(data).astype(float)
    y = np.array(y).astype(int)
    print(test(weight_final, data, y))
    print(weight_final)
    return weight_final
pass
def Virtualization(data, y, weight_final): #Virtualization the scatter plot and draw the classification line,ONLY WORKS IN 2D
    x0,y0,x1,y1 = [],[],[],[]
    data = np.array(data).astype(float)
    y = np.array(y).astype(int)#transfer data into numerical foramt
    for i in range(0,n):#classify two kinds of data
        if y[i] == 1:
            x0.append(data[i][0])
            y0.append(data[i][1])
        else:
            x1.append(data[i][0])
            y1.append(data[i][1])
    fig = plt.figure()
    ax = fig.add_subplot(1,1, 1)
    #print(x0, y0)
    ax.set_title("Logic Regression of Melon Example")
    ax.set_xlabel("Density of melons")
    ax.set_ylabel("Sugar content")
    ax.scatter(x0,y0, label = 'Bad Melons', marker = 'o')#Plot scatter
    ax.scatter(x1,y1,  label = 'Good Melons', marker = 's')
    weight_final = np.array(weight_final)
    X = []
    Y = []
    for i in [1,9]:
        X.append(i*0.1)
        Y.append(-(i*0.1*weight_final[0][0] + weight_final[0][2])/weight_final[0][1])
    print (X,Y)
    ax.plot(X,Y, label = 'classification line')#Plot classification line
    ax.legend()
    plt.show()

file = open("data.txt")
data = []
y = []
n = 0
while 1:#read data
    line = file.readline().split()
    if not line:
        break
    pass
    #line = line.split(' ')
    k = len(line)
    #print (k, data, y)
    lines = []
    for i in range(1,k-1):
        lines.append(line[i])
    lines.append(1)
    data.append(lines)
    if line[-1] == '0':
        y.append(0)
    else:
        y.append(1)
    n += 1

print(data, y)
weight_final = kFoldeCrossValidation(10, data, y)

Virtualization(data, y, weight_final)


from validate import validate_article_row
from model.articles import Articles
from peewee import IntegrityError
from logger import logging
from datetime import datetime

class Scraper:
    def __init__(self, name):
        self.name = name

    def fetch(self):
        raise NotImplemented

    @staticmethod
    def insert_article(article_row):
        validate_article_row(article_row)

        # insert into db
        q = Articles.insert(**article_row)
        try:
            q.execute()
        except IntegrityError:
            logging.debug('Skipping duplicate entry: {0}, {1}'.format(article_row['source'], article_row['date_pub']))

trend_stop_words = []


import requests
import json
from queue import Queue
import os
import time
from lxml import etree
import threading
import pymongo
import urllib3
from retrying import retry
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import re

class SinaNews:
    def __init__(self):
        self.headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Host": "feed.mix.sina.com.cn",
            "Proxy-Connection": "keep-alive",
            "Referer": "http://tech.sina.com.cn/chuangshiji/",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
        }
        self.details_headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Host": "tech.sina.com.cn",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
        }
        self.proxies = {
            "https": "120.24.68.197:16818",
            "http": "120.24.68.197:16818"
        }
        self.MONGO_URL = "192.168.0.8"
        self.MONGO_PORT = 27017
        self.MONGO_DB = "sina_news"
        self.MONGO_COLLECTION = "sina_news_list"
        self.client = pymongo.MongoClient(self.MONGO_URL)
        self.db = self.client[self.MONGO_DB]
        self.coll = self.db[self.MONGO_COLLECTION]
        self.url_queue = Queue()
        self.cy_url_queue = Queue()
        self.content = Queue()
        self.cy_content = Queue()
        self.data = Queue()
        self.image_news_queue = Queue()

    @retry(stop_max_attempt_number=4)
    def _parse_url(self, url, headers, return_str=False):
        r = requests.get(url, headers=headers, proxies=self.proxies, timeout=5)
        # assert r.status_code == 200
        return r.content.decode("utf-8")

    # @retry(stop_max_attempt_number=5)
    def get_news_list(self):
        for i in range(1):
            start_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=223&lid=23&num=50&versionNumber=1.2.8&page{}"
            response = self._parse_url(start_url.format(i), headers=self.headers)
            item_url_dict = json.loads(response)["result"]["data"]

            for k in item_url_dict:
                # print(k)
                details_url = (k["url"]).replace('["', "").replace('"]', "").replace('\\', "")
                if "internet" not in details_url or "http://slide.tech.sina.com.cn/" not in details_url:
                    self.url_queue.put(details_url)

    # @retry(stop_max_attempt_number=5)
    def get_details_page(self):
        """chuang shi ji """
        while True:
            page_url = self.url_queue.get()
            print(page_url)
            if "csj" in page_url:
                # print(page_url)
                html = self._parse_url(url=page_url, headers=self.details_headers)
                print(page_url)
                time.sleep(1)
                self.content.put(html)
            else:
                # print(page_url)
                html = self._parse_url(url=page_url, headers=self.details_headers)
                print(page_url)
                time.sleep(1)
                self.cy_content.put(html)

            self.url_queue.task_done()

    def parse_cy_item(self):
        while True:
            doc = self.cy_content.get()
            text = etree.HTML(doc)
            news_title = text.xpath('//div[@class="second-title"]/text()') if text.xpath(
                '//div[@class="second-title"]/text()') else None
            if news_title is not None:
                news_title = news_title[0]

            news_date = text.xpath('//div[@class="date-source"]/span[1]/text()') if text.xpath(
                '//div[@class="date-source"]/span[1]/text()') else None
            if news_date is not None:
                news_date = news_date[0]

            news_author = text.xpath('//div[@class="date-source"]/span[2]/text()') if text.xpath(
                '//div[@class="date-source"]/span[2]/text()') else None
            if news_author is not None:
                news_author = news_author[0]
            else:
                news_author = text.xpath('//div[@class="date-source"]/span[2]/a/text()') if text.xpath(
                    '//div[@class="date-source"]/span[2]/a/text()') else None
                if news_author is not None:
                    news_author = news_author[0]

            news_tag = text.xpath('//div[@id="keywords"]/a/text()') if text.xpath(
                '//div[@id="keywords"]/a/text()') else []

            news_url = text.xpath('//meta[@property="toutiao"]/@content') if text.xpath(
                '//meta[@property="toutiao"]/@content') else None
            if news_url is not None:
                news_url = news_url[0]

            news_cate = text.xpath('//div[@class="path"]//span[@class="spliter"]/a/text()') if text.xpath(
                '//div[@class="path"]//span[@class="spliter"]/a/text()') else None
            if news_cate is not None:
                news_cate = news_cate[0]

            news_content = text.xpath('//div[@id="artibody"]') if text.xpath('//div[@id="artibody"]') else None
            if news_content is not None:
                news_content = news_content[0]
            content = etree.tostring(news_content, encoding="utf-8").decode("utf-8")

            data = {
                "title": news_title,
                "time": news_date,
                "categories": news_cate,
                "url": news_url,
                "author": news_author,
                "tag": news_tag,
                "content": content,
            }
            self.data.put(data)
            self.cy_content.task_done()

    def parse_item(self):
        """
        1. 创事记页面解析
        :return:
        """
        while True:
            doc = self.content.get()
            text = etree.HTML(doc)
            news_title = text.xpath('//h1[@id="artibodyTitle"]/text()') if text.xpath(
                '//h1[@id="artibodyTitle"]/text()') else None
            if news_title is not None:
                news_title = news_title[0]

            news_date = text.xpath('//span[@id="pub_date"]/text()') if text.xpath(
                '//span[@id="pub_date"]/text()') else None
            if news_date is not None:
                news_date = news_date[0].strip()

            news_cate = text.xpath('//span[@id="media_name"]/a[1]/text()') if text.xpath(
                '//span[@id="media_name"]/a[1]/text()') else None
            if news_cate is not None:
                news_cate = news_cate[0]

            news_author = text.xpath('//span[@id="author_ename"]/a/text()') if text.xpath(
                '//span[@id="author_ename"]/a/text()') else None
            if news_author is not None:
                news_author = news_author[0]

            news_tag = text.xpath('//p[@class="art_keywords"]/a/text()') if text.xpath(
                '//p[@class="art_keywords"]/a/text()') else None
            if news_tag is not None:
                news_tag = news_tag

            news_url = text.xpath('//div[@class="blk_tw"]/h3/a/@href') if text.xpath(
                '//div[@class="blk_tw"]/h3/a/@href') else None
            if news_url is not None:
                news_url = news_url[0]
            else:
                news_url = text.xpath('//meta[@property="toutiao"]/@content') if text.xpath(
                    '//meta[@property="toutiao"]/@content') else None
                if news_url is not None:
                    news_url = news_url[0]

            news_content = text.xpath('//div[@id="artibody"]') if text.xpath('//div[@id="artibody"]') else None
            if news_content is not None:
                news_content = news_content[0]
            content = etree.tostring(news_content, encoding="utf-8").decode("utf-8")

            data = {
                "title": news_title,
                "time": news_date,
                "categories": news_cate,
                "url": news_url,
                "author": news_author,
                "tag": news_tag,
                "content": content,
            }
            self.data.put(data)
            self.content.task_done()

    def save_data(self):
        while True:
            data = self.data.get()
            try:
                if self.db[self.MONGO_COLLECTION].insert(data):
                    print('存储到MongoDB成功')
            except Exception as e:
                print(e)
                print('存储到MongoDB失败')
            self.data.task_done()

    def run(self):
        self.get_news_list()

        thread_list = []
        for i in range(1):
            t2 = threading.Thread(target=self.get_details_page)
            thread_list.append(t2)

        # for i in range(1):
        #     t3 = threading.Thread(target=self.parse_item)
        #     thread_list.append(t3)
        #
        # for i in range(1):
        #     t4 = threading.Thread(target=self.parse_cy_item)
        #     thread_list.append(t4)
        #
        # for i in range(1):
        #     t5 = threading.Thread(target=self.save_data)
        #     thread_list.append(t5)
        #
        # for t in thread_list:
        #     t.setDaemon(True)  # 设置守护线程，说明该线程不重要，主线程结束，子线程结束
        #     t.start()  # 线程启动
        #
        # for q in [self.url_queue, self.content, self.data]:
        #     q.join()  # 等待，让主线程等待，队列计数为0之后才会结束，否则会一直等待

if __name__ == '__main__':
    sn = SinaNews()
    sn.run()


from autoslug import AutoSlugField
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from tinymce.models import HTMLField

from raporty_siecobywatelska_pl.ranking.models import Ranking

@python_2_unicode_compatible
class Group(models.Model):
    name = models.CharField(
        max_length=250,
        verbose_name=_("Name")
    )
    description = HTMLField(
        verbose_name=_("Description")
    )
    slug = AutoSlugField(
        populate_from='name',
        verbose_name=_("Slug"),
        unique=True
    )
    ranking = models.ForeignKey(
        to=Ranking,
        on_delete=models.CASCADE
    )

    class Meta:
        verbose_name = _("Question group")
        verbose_name_plural = _("Question groups")
        ordering = ['name']

    def get_absolute_url(self):
        return reverse('questionnaire:ranking-group-detail', args=[self.ranking.slug, self.slug])

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Question(models.Model):
    name = models.CharField(max_length=250, verbose_name=_("Name"))
    description = HTMLField(verbose_name=_("Description"))

    group = models.ForeignKey(Group, on_delete=models.CASCADE)

    class Meta:
        verbose_name = _("Question")
        verbose_name_plural = _("Question")
        ordering = ['name']

    def get_absolute_url(self):
        return reverse('rankings:detail', kwargs={'slug': self.group.ranking.slug})

    def __str__(self):
        return self.name


#!/usr/bin/python
# -*- coding: utf-8 -*-
""" A Tank Game to teach coding those unfamiliar with it"""

import pygame
import sys
from pygame.locals import *
from PIL import Image
import random
import time

pygame.init()

# Set the width and height of the screen

infoObject = pygame.display.Info()
size = (infoObject.current_w, infoObject.current_h - 30)

# Used to manage how fast the screen updates

clock = pygame.time.Clock()
class tank:

    def __init__(self):
        self.origional_image = pygame.image.load('KV-2_Tank.png')
        self.image = self.origional_image
        self.rotation = 0
        self.coords = (0, 0)
        self.move_count = 0  # # Counts the number of tims a function is iterated

    def rotate(self, degrees):
        """ This program will rotate 1 degree everytime called, untill it has
            reached the desired rotation, then it will advance the program counter
        """

        global program_counter

        if degrees > 0 and self.move_count < degrees:
            self.rotation += 1
            self.move_count += 1
        elif degrees < 0 and self.move_count > degrees:
            self.rotation -= 1
            self.move_count -= 1
        else:
            self.move_count = 0
            program_counter += 1

            # # Converting large negetive rotations to the corresponding small positive ones

            if abs(self.rotation) > 180:
                if self.rotation < 0:
                    self.rotation = 360 + self.rotation
                elif self.rotation > 0:
                    self.rotation = -360 + self.rotation

    def move(self, distance):
        """ This program will move 10 pixels everytime called, untill it has
            reached the desired position, then it will advance the program counter
        """

        global program_counter

        self.move_count += 1
        (x, y) = self.coords

        if distance < 0:
            factor = -1
        elif distance == 0:
            factor = 0
        else:
            factor = 1

        if self.move_count < abs(distance):
            y += factor * -(5 - abs(self.rotation) / 18)
            if abs(self.rotation) <= 90:
                x -= factor * self.rotation / 18
            elif self.rotation > 0:
                x -= factor * (180 - self.rotation) / 18
            else:
                x -= factor * -(180 - abs(self.rotation)) / 18
        else:

            program_counter += 1
            self.move_count = 0

        self.coords = (x, y)

    def fire(self):
        """ Fires a bullet from the tank"""

        global program_counter
        global bullet_list
        x,y = self.coords
        bullet_list.append(bullet((x, y- 1/2 * self.image.get_height()), self.rotation))
        program_counter += 1

    def delay(self, milliseconds):
        """Delays the tank's movement"""

        global program_counter
        time.sleep(milliseconds * 0.001)
        program_counter += 1

    def draw(self):
        """ Draws the image of the tank on the playing surface"""

        self.image = _rot_center(self.origional_image, self.rotation)
        screen.blit(self.image, self.coords)

class bullet:

    def __init__(self, coordinates, rotation):
        self.origional_image = \
            pygame.transform.scale(pygame.image.load('bullet.png'),
                                   (20, 10))
        self.image = self.origional_image
        self.rotation = rotation
        self.coords = coordinates
        self.move_count = 0  ## Counts the number of tims a function is iterated

    def move(self, distance):
        """ This program will move 10 pixels everytime called, untill it has
        reached the desired position, then it will advance the program counter
        """

        self.move_count += 1
        (x, y) = self.coords

        if distance < 0:
            factor = -1
        elif distance == 0:
            factor = 0
        else:
            factor = 1

        if self.move_count < abs(distance):
            y += factor * -(5 - abs(self.rotation) / 18)
            if abs(self.rotation) <= 90:
                x -= factor * self.rotation / 18
            elif self.rotation > 0:
                x -= factor * (180 - self.rotation) / 18
            else:
                x -= factor * -(180 - abs(self.rotation)) / 18
        else:

            self.move_count = 0

        self.coords = (x, y)

    def draw(self):
        """ Draws the image of the tank on the playing surface"""

        self.image = _rot_center(self.origional_image, self.rotation
                                 + 90)
        screen.blit(self.image, self.coords)

def _rot_center(pygame_image, degrees):
    """ Rotate an image while keeping its center and size, by converting to pil, rotating and back"""

    image_string = pygame.image.tostring(pygame_image, 'RGBA', False)
    PIL_image = Image.frombytes('RGBA', pygame_image.get_size(),
                                image_string)
    PIL_image = PIL_image.rotate(degrees)
    image_string = PIL_image.tobytes()
    pygame_image = pygame.image.fromstring(image_string,
            PIL_image.size, 'RGBA')
    return pygame_image

def get_program(nest):
    """ This function gets the tank program from the user"""

    user_in = True
    program = []

    # # Gets program from user

    while user_in:
        user_in = input('>>> ' + nest * '  ')
        if 'tank.' in user_in:
            user_in = 'tank_program.append("' + user_in + '")'

        # # If a conditional, it runs 'get_program' to get the conditional
        # # the code under the conditional,and appends it as one string
        # # separated by '\n' to the program list. It counts the number of
        # # 'nests' to print the correct number of tabs.

        if ':' in user_in:
            nested_program = get_program(nest + 1)
            for command in nested_program:
                user_in += '\n' + (nest + 1) * '  ' + command
        program.append(user_in)
    return program

def start(program):
    """ This function is called, given the input data, collected from the user
        and processed by 'get_program'. Given this list of commands, the computer
        moves the tank on the screen.
    """

    # # Calculating the moves of the Tank. Compiling

    global program_counter
    while True:
        if program_counter < len(program) - 1:
            exec(program[program_counter])
            program_counter += 1
        else:
            break

    # # Executing the moves of the tank. Interpreting

    program_counter = 0
    while True:
        if program_counter < len(tank_program):
            print(tank_program[program_counter])
            exec(tank_program[program_counter])
        screen.fill((255, 255, 255))
        for bullet in bullet_list:
            bullet.move(10)
            bullet.draw()

        tank.draw()
        pygame.display.flip()
        time.sleep(0.001)
        clock.tick(60)

program = get_program(0)
tank_program = []
program_counter = 0
tank = tank()
bullet_list = []

screen = pygame.display.set_mode(size)  # , FULLSCREEN)
start(program)
# Close the window and quit.
pygame.quit()

Tank Game/Outdated/Tank Game compiled.py


"""
  Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

  Permission is hereby granted, free of charge, to any person obtaining a copy of this
  software and associated documentation files (the "Software"), to deal in the Software
  without restriction, including without limitation the rights to use, copy, modify,
  merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
  INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
  PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
from cfnlint import CloudFormationLintRule
from cfnlint import RuleMatch
import cfnlint.helpers

class Exclusive(CloudFormationLintRule):
    """Check Properties Resource Configuration"""
    id = 'E2520'
    shortdesc = 'Check Properties that are mutually exclusive'
    description = 'Making sure CloudFormation properties ' + \
                  'that are exclusive are not defined'
    tags = ['base', 'resources']

    def __init__(self):
        """Init"""
        self.exlusivespec = cfnlint.helpers.load_resources('data/ResourcePropertiesExclusive.json')

    def match(self, cfn):
        """Check CloudFormation Properties"""
        matches = list()

        for excl_type, excl_values in self.exlusivespec.items():
            for res_name, res_value in cfn.get_resources(excl_type).items():
                for excl_name, excl_value in excl_values.items():
                    properties = res_value.get('Properties', {})
                    if excl_name in properties:
                        for prop_name in excl_value:
                            if prop_name in res_value['Properties']:
                                message = "Parameter {0} shouldn't exist with {1} for {2}"
                                matches.append(RuleMatch(
                                    ['Resources', res_name, 'Properties', excl_name],
                                    message.format(excl_name, prop_name, res_name)
                                ))

        return matches