import scrapy
import subprocess
import json
import os

def installPackage(package):
    try:
        return __import__(str(package))
    except ImportError:
        subprocess.call(["pip3", "install", "scrapy"])

# Make a dicrectory to contain JSON file and HTML file
class makeDir():
    def JSON_dir(self):
        JSON_path = os.path.join(os.getcwd(), "JSON Image file/")

        if not os.path.exists(JSON_path):
            os.mkdir(JSON_path)

        return JSON_path

    def HTML_dir(self):
        HTML_path = JSON_path = os.path.join(os.getcwd(), "HTML File/")

        if not os.path.exists(HTML_path):
            os.mkdir(HTML_path)

        return HTML_path

dir = makeDir()
JSON_directory = dir.JSON_dir()
HTML_directory = dir.HTML_dir()

# For dantri.com
class covid_dantri(scrapy.Spider):
    name = "covid_dantri"

    def start_requests(self):
        urls = [
            "https://dantri.com.vn/suc-khoe/dai-dich-covid-19.htm",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-1].split(".")[0].split(".")[0]
        filename = f'{HTML_directory}dantri-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        # Writing source and caption of image into JSON file
        dantri_covid = {'img': []}

        for image in response.css("img"):
            dantri_covid['img'].append({
                "source": image.attrib["src"],
                "caption": image.attrib["alt"],
            })

        with open(f"{JSON_directory}/image_dantri.json", "w") as image_file:
            json.dump(dantri_covid, image_file)

# For tuoitre.net
class covid_tuoitre(scrapy.Spider):
    name = "covid_tuoitre"

    def start_requests(self):
        urls = [
            "https://tuoitre.vn/phong-chong-covid-19-e583.htm",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-1].split(".")[0].split(".")[0]
        filename = f'{HTML_directory}tuoitre-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        tuoitre_covid = {'img': []}

        source = response.css('img::attr(src)').extract()
        caption = response.css('img::attr(alt)').extract()

        for i in range(0, len(caption)):
            tuoitre_covid['img'].append({
                "source": source[i],
                "caption": caption[i],
            })

        with open(f"{JSON_directory}image_tuoitre.json", "w") as image_file:
            json.dump(tuoitre_covid, image_file)

# For vnexpress.net
class covid_vnexpress(scrapy.Spider):
    name = "covid_vnexpress"

    def start_requests(self):
        urls = [
            "https://vnexpress.net/covid-19/tin-tuc",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = f'{HTML_directory}vnexpress-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

        # Writing source and caption of image into JSON file
        vnexpress_covid = {'img': []}

        for image in response.css("img"):
            vnexpress_covid['img'].append({
                "source": image.attrib["src"],
                "caption": image.attrib["alt"],
            })

        with open(f"{JSON_directory}image_vnexpress.json", "w") as image_file:
            json.dump(vnexpress_covid, image_file)

def main():
    # Install Scrapy if not have
    installPackage(scrapy)

    # Crawl image and caption about Covid19 News
    # From dantri.com
    subprocess.call(["scrapy", "crawl", "covid_dantri"])

    # From tuoitre.net
    subprocess.call(["scrapy", "crawl", "covid_tuoitre"])

    # From vnexpress.net
    subprocess.call(["scrapy", "crawl", "covid_vnexpress"])

if __name__ == "__main__":
    main()


import collections
import heapq
from typing import List
import time

class Solution:
    def findKthLargest(self, nums: List[int], k: int) -> int:
        nums.sort(reverse=True)
        maxK = 0
        for i in range(0, k):
            maxK = nums[i]
        return maxK

def main():
    sol = Solution()
    inputs = [[3,2,1,5,6,4], [3,2,3,1,2,4,5,5,6]]
    ks = [2, 4]
    outputs = [5, 4]

    for i in range(len(inputs)):
        start = time.time()
        res = sol.findKthLargest(inputs[i], ks[i])
        end = time.time()
        if outputs[i] == res:
            print("true : ", end-start, " sec")
        else:
            print("false : ", end-start, " sec")

if __name__ == "__main__":
    main()


import os
import shutil
import copy
import pickle
import math
import skimage
import sklearn
import scipy
import numpy as np

from collections import defaultdict
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse

import skimage.io as io
from skimage.transform import rescale
from skimage.feature import blob_log
from skimage.morphology import closing, square
from skimage.filters import threshold_otsu
from skimage.measure import label

import skan
from skan import skeleton_to_csgraph

from sklearn import preprocessing
from sklearn import linear_model
from sklearn import decomposition
from sklearn.cluster import KMeans

class Cell:
    """
    Extract the individual cell by thresholding and removing background noise.
    """

    def __init__(self, cell_image, image_type, reference_image=None):
        """
        Args:

        cell_image: RGB cell image
        image_type: 'confocal' or 'DAB'
        reference_image: all images can be standardised to the exposure level of this example cell image

        """

        self.cell_image = cell_image
        self.image_type = image_type
        self.gray_cell_image = skimage.color.rgb2gray(self.cell_image)
        self.inverted_gray_cell_image = skimage.util.invert(self.gray_cell_image)
        self.reference_image = reference_image
        self.threshold_image = self.threshold_image()
        self.inverted_threshold_image = skimage.util.invert(self.threshold_image)
        self.cleaned_image = self.remove_small_object_noise()
        self.cleaned_image_filled_holes = self.fill_holes()

    def get_blobs(self):
        # Extracts circular blobs in cell image for finding soma later

        if self.image_type == "DAB":
            blobs_log = blob_log(self.inverted_gray_cell_image, min_sigma=6, max_sigma=20, num_sigma=10, threshold=0.1, overlap=0.5)
        elif self.image_type == "confocal":
            blobs_log = blob_log(self.cell_image, min_sigma=3, max_sigma=20, num_sigma=10, threshold=0.1, overlap=0.5)

            def eliminate_border_blobs(blobs_log):
                # find the blobs too close to border so as to eliminate them
                blobs_dict = defaultdict()
                for i, blob in enumerate(blobs_log):
                    blobs_dict[i] = np.take(blob, [0,1,3])
                    y, x, r = blobs_dict[i]
                    image_border_x, image_border_y = self.cell_image.shape[1]/5, self.cell_image.shape[0]/5
                    if x < image_border_x or x > 4*image_border_x or y < image_border_y or y > 4*image_border_y:
                        blobs_dict.pop(i)
                blobs_log=[]
                for key, blobs in blobs_dict.items():
                    blobs_log.append(blobs)
                return blobs_log

            blobs_log = eliminate_border_blobs(blobs_log)

            if len(blobs_log)<1:
                # if none of the blobs remain after border blob elimination, try blob_log with less stringent parameters
                blobs_log = blob_log(self.cell_image, min_sigma=2, max_sigma=20, num_sigma=10, threshold=0.1, overlap=0.5)
                blobs_log = eliminate_border_blobs(blobs_log)

        return blobs_log

    def centre_of_mass(self, blobs):
        # finds centre of mass of the multiple blobs detected

        # find the blob with highest intensity value
        ixs = np.indices(self.gray_cell_image.shape)

        blob_intensities=[]
        blob_centres=[]
        blob_radiuses=[]

        for blob in blobs:
            y, x, r = blob
            # Define an array of shape `[2, 1, 1]`, containing the center of the blob
            blob_center = np.array([y, x])[:, np.newaxis, np.newaxis]
            # Using the formula for a circle, `x**2 + y**2 < r**2`, generate a mask for this blob.
            mask = ((ixs - blob_center)**2).sum(axis=0) < r**2
            # Calculate the average intensity of pixels under the mask
            blob_avg_est = self.gray_cell_image[mask].mean()
            blob_intensities.append(blob_avg_est)
            blob_centres.append((y, x))
            blob_radiuses.append(r)

        if self.image_type == "DAB":
            max_intensity = blob_centres[np.argmin(blob_intensities)]

            return max_intensity

        elif self.image_type == "confocal":
            max_radius = blob_centres[np.argmax(blob_radiuses)]
            max_intensity = blob_centres[np.argmax(blob_intensities)]
            if len(blob_radiuses) > len(set(blob_radiuses)):
                return max_intensity
            else:
                return max_radius

    def get_soma(self):
        # calculate pixel position to be attribute as soma

        soma_blobs = self.get_blobs()

        if len(soma_blobs)==1:
            soma = soma_blobs[0][:2]
        if len(soma_blobs)>1:
            soma = self.centre_of_mass(soma_blobs)

        return soma

    def threshold_image(self):
        if self.reference_image is not None:
            self.gray_reference_image = skimage.color.rgb2gray(self.reference_image)
            self.gray_cell_image = skimage.transform.match_histograms(self.gray_cell_image, self.gray_reference_image)

        # Contrast stretching
        p2, p98 = np.percentile(self.gray_cell_image, (2, 98))
        img_rescale = skimage.exposure.rescale_intensity(self.gray_cell_image, in_range=(p2, p98))

        thresholded_cell = img_rescale > threshold_otsu(img_rescale)

        if self.image_type == "DAB":
            return thresholded_cell
        elif self.image_type == "confocal":
            return skimage.util.invert(thresholded_cell)

    def label_objects(self):
        bw = closing(self.inverted_threshold_image, square(1))
        # label image regions
        labelled_image, no_of_objects = skimage.measure.label(bw, return_num=True)

        return labelled_image, no_of_objects

    def remove_small_object_noise(self):
        labelled_image, no_of_objects = self.label_objects()
        object_areas = []
        for object_label in range(1, no_of_objects+1):
        # for object_label in range(no_of_objects):
            object_areas.append(len(np.where(labelled_image==[object_label])[0]))

        largest_object_label = np.argmax(object_areas)+1
        astrocyte_image = np.where(labelled_image==[largest_object_label], 1, 0)

        return astrocyte_image

    def fill_holes(self):
        return scipy.ndimage.binary_fill_holes(self.cleaned_image).astype(int)

    def surface_area(self):
        return np.sum(self.cleaned_image)

class Skeleton:
    """
    Skeletonize the thresholded image and extract relevant features from the 2D skeleton.
    """

    def __init__(self, cell_image, image_type):
        """
        Args:

        cell_image: RGB cell image
        image_type: 'confocal' or 'DAB'

        """

        self.cell_image = cell_image
        self.astrocyte = Cell(cell_image, image_type)
        self.cleaned_image = self.astrocyte.cleaned_image_filled_holes
        self.soma = self.astrocyte.get_soma()
        self.cell_skeleton = self.skeletonization()
        self.soma_on_skeleton = self.get_soma_on_skeleton()
        self.padded_skeleton = self.pad_skeleton()
        self.classify_branching_structure()

    def distance(self, P1, P2):
        # find eucledian distance between two pixel positions
        return ((P1[0] - P2[0])**2 + (P1[1] - P2[1])**2) ** 0.5

    def skeletonization(self):
        # perform skeletonization
        return skimage.morphology.skeletonize(self.cleaned_image)

    def pad_skeleton(self):

        # get all the pixel indices representing skeleton
        skeleton_indices = np.nonzero(self.cell_skeleton)

        # get corner points enclosing skeleton
        x_min, x_max = min(skeleton_indices[1]), max(skeleton_indices[1])
        y_min, y_max = min(skeleton_indices[0]), max(skeleton_indices[0])
        self.bounded_skeleton = self.cell_skeleton[y_min:y_max, x_min:x_max]

        pad_width = max(self.bounded_skeleton.shape)//2
        self.bounded_skeleton_boundary = [x_min, x_max, y_min, y_max]

        # get updated soma position on bounded and padded skeleton
        self.soma_on_bounded_skeleton = self.soma_on_skeleton[0]-y_min, self.soma_on_skeleton[1]-x_min
        self.soma_on_padded_skeleton = self.soma_on_skeleton[0]-y_min+pad_width, self.soma_on_skeleton[1]-x_min+pad_width

        return skimage.util.pad(self.bounded_skeleton, pad_width=pad_width, mode='constant')

    def get_soma_on_skeleton(self):
        skeleton_pixel_coordinates = [(i, j) for (i, j), val in np.ndenumerate(self.cell_skeleton) if val!=0]
        soma_on_skeleton = min(skeleton_pixel_coordinates, key=lambda x: self.distance(self.soma, x))

        return soma_on_skeleton

    def total_length(self):
        return np.sum(self.cell_skeleton)

    def avg_process_thickness(self):
        return round((self.astrocyte.surface_area()/self.total_length()), 1)

    def convex_hull(self, plot=False):
        convex_hull = skimage.morphology.convex_hull_image(self.cell_skeleton)
        if plot==True:
            fig, ax = plt.subplots()
            ax.set_axis_off()
            ax.imshow(convex_hull)

        return np.sum(convex_hull)

    def get_no_of_forks(self, plot=False):

        # get the degree for every cell pixel (no. of neighbouring pixels)
        pixel_graph, coordinates, degrees = skeleton_to_csgraph(self.cell_skeleton)
        # array of all pixel locations with degree more than 2
        fork_image = np.where(degrees > [2], 1, 0)
        s = scipy.ndimage.generate_binary_structure(2,2)
        labeled_array, num_forks = scipy.ndimage.label(fork_image, structure=s)

        if plot==True:
            fork_indices = np.where(degrees > [2])
            fork_coordinates = zip(fork_indices[0], fork_indices[1])

            fig, ax = plt.subplots(figsize=(4, 4))
            ax.set_title('path')
            ax.imshow(self.cell_skeleton, interpolation='nearest')

            for i in fork_coordinates:
                c = plt.Circle((i[1], i[0]), 0.6, color='green')
                ax.add_patch(c)

            ax.set_axis_off()
            plt.tight_layout()
            plt.show()

        return num_forks

    def eliminate_loops(self, branch_statistics, paths_list):
        loop_indexes=[]
        loop_branch_end_points=[]

        # set that keeps track of what elements have been added
        seen = set()
        # eliminate loops from branch statistics
        for branch_no, branch in enumerate(branch_statistics):
            # If element not in seen, add it to both
            current = (branch[0], branch[1])
            if current not in seen:
                seen.add(current)
            elif current in seen:
                # for deleting the loop index from branch statistics
                loop_indexes.append(branch_no)
                # for deleting the paths from paths list by recognizing loop end points
                loop_branch_end_points.append((int(branch[0]), int(branch[1])))

        new_branch_statistics = np.delete(branch_statistics, loop_indexes, axis=0)

        # eliminate loops from paths list
        path_indexes=[]
        for loop_end_points in loop_branch_end_points:
            for path_no, path in enumerate(paths_list):
                if loop_end_points[0]==path[0] and loop_end_points[1]==path[-1] or loop_end_points[0]==path[-1] and loop_end_points[1]==path[0]:
                    path_indexes.append(path_no)
                    break

        new_paths_list = np.delete(np.array(paths_list), path_indexes, axis=0)

        return new_branch_statistics, new_paths_list

    def branch_structure(self, junctions, branch_statistics, paths_list):
        next_set_junctions = []
        next_set_branches = []
        terminal_branches=[]

        for junction in junctions:
            branches_travelled = []
            for branch_no, branch in enumerate(branch_statistics):
                if branch[0]==junction:
                    if branch[3]==2:
                        next_set_junctions.append(branch[1])
                        for path in paths_list:
                            if branch[0]==path[0] and branch[1]==path[-1] or branch[0]==path[-1] and branch[1]==path[0]:
                                next_set_branches.append(path)
                                branches_travelled.append(branch_no)
                    if branch[3]==1:
                        for path in paths_list:
                            if branch[0]==path[0] and branch[1]==path[-1] or branch[0]==path[-1] and branch[1]==path[0]:
                                terminal_branches.append(path)
                                next_set_branches.append(path)
                                branches_travelled.append(branch_no)
                elif branch[1]==junction:
                    if branch[3]==2:
                        next_set_junctions.append(branch[0])
                        for path in paths_list:
                            if branch[0]==path[0] and branch[1]==path[-1] or branch[0]==path[-1] and branch[1]==path[0]:
                                next_set_branches.append(path)
                                branches_travelled.append(branch_no)
                    if branch[3]==1:
                        for path in paths_list:
                            if branch[0]==path[0] and branch[1]==path[-1] or branch[0]==path[-1] and branch[1]==path[0]:
                                terminal_branches.append(path)
                                next_set_branches.append(path)
                                branches_travelled.append(branch_no)
            branch_statistics = np.delete(branch_statistics, branches_travelled, axis=0)

        return next_set_junctions, next_set_branches, terminal_branches, branch_statistics

    def classify_branching_structure(self, plot=False):

        def get_soma_node():
            near = []
            for i in range(skan.csr.Skeleton(self.cell_skeleton).n_paths):
                path_coords = skan.csr.Skeleton(self.cell_skeleton).path_coordinates(i)
                nearest = min(path_coords, key=lambda x: self.distance(self.soma_on_skeleton, x))
                near.append(nearest)

            soma_on_path = min(near, key=lambda x: self.distance(self.soma_on_skeleton, x))

            for i,j in enumerate(skan.csr.Skeleton(self.cell_skeleton).coordinates):
                if all(soma_on_path==j):
                    soma_node = [i]
                    break

            return soma_node

        def get_soma_branches(soma_node, paths_list):
            soma_branches=[]
            for path in paths_list:
                # print(path)
                if soma_node in path:
                    soma_branches.append(path)
            return soma_branches

        pixel_graph, coordinates, degrees = skeleton_to_csgraph(self.cell_skeleton)
        branch_statistics = skan.csr.branch_statistics(pixel_graph)
        paths_list = skan.csr.Skeleton(self.cell_skeleton).paths_list()

        terminal_branches = []
        branching_structure_array = []
        # get branches containing soma node

        soma_node = get_soma_node()
        soma_branches = get_soma_branches(soma_node, paths_list)
        if len(soma_branches)>2:
            junctions = soma_node
            delete_soma_branch=False
        else:
            # collect first level/primary branches
            junctions = [soma_branches[0][0], soma_branches[0][-1]]
            delete_soma_branch=True

        # eliminate loops in branches and path lists
        branch_statistics, paths_list = self.eliminate_loops(branch_statistics, paths_list)

        while True:
            junctions, branches, terminal_branch, branch_statistics = self.branch_structure(junctions, branch_statistics, paths_list)
            branching_structure_array.append(branches)
            terminal_branches.extend(terminal_branch)
            if len(junctions)==0:
                break

        if delete_soma_branch==True:
            branching_structure_array[0].remove(soma_branches[0])

        if plot==True:
            # store same level branch nodes in single array
            color_branches_coords=[]
            for branch_level in branching_structure_array:
                single_branch_level=[]
                for path in branch_level:
                    path_coords=[]
                    for node in path:
                        path_coords.append(coordinates[node])
                        single_branch_level.extend(path_coords)
                color_branches_coords.append(single_branch_level)

            fig, ax = plt.subplots(figsize=(4, 4))
            ax.set_title('path')
            ax.imshow(self.cell_skeleton, interpolation='nearest')

            color_codes = ['red', 'blue', 'magenta', 'green', 'cyan']
            for j, color_branch in enumerate(color_branches_coords):
                if j>4:
                    j=4
                for k in color_branch:
                    c = plt.Circle((k[1], k[0]), 0.5, color=color_codes[j])
                    ax.add_patch(c)

            ax.set_axis_off()
            plt.tight_layout()
            plt.show()

        self.branching_structure_array = branching_structure_array
        self.terminal_branches = terminal_branches

    def get_primary_branches(self):
        primary_branches = self.branching_structure_array[0]
        no_of_primary_branches = len(primary_branches)
        avg_length_of_primary_branches = 0 if no_of_primary_branches == 0 else sum(map(len, primary_branches))/float(len(primary_branches))

        return primary_branches, no_of_primary_branches, round(avg_length_of_primary_branches, 1)

    def get_secondary_branches(self):
        try:
            secondary_branches = self.branching_structure_array[1]
        except IndexError:
            secondary_branches=[]
        no_of_secondary_branches = len(secondary_branches)
        avg_length_of_secondary_branches = 0 if no_of_secondary_branches == 0 else sum(map(len, secondary_branches))/float(len(secondary_branches))

        return secondary_branches, no_of_secondary_branches, round(avg_length_of_secondary_branches, 1)

    def get_tertiary_branches(self):
        try:
            tertiary_branches = self.branching_structure_array[2]
        except IndexError:
            tertiary_branches=[]
        no_of_tertiary_branches = len(tertiary_branches)
        avg_length_of_tertiary_branches = 0 if no_of_tertiary_branches == 0 else sum(map(len, tertiary_branches))/float(len(tertiary_branches))

        return tertiary_branches, no_of_tertiary_branches, round(avg_length_of_tertiary_branches, 1)

    def get_quatenary_branches(self):
        try:
            quatenary_branches = self.branching_structure_array[3:]
        except IndexError:
            quatenary_branches=[]
        quatenary_branches = [branch for branch_level in quatenary_branches for branch in branch_level]
        no_of_quatenary_branches = len(quatenary_branches)
        avg_length_of_quatenary_branches = 0 if no_of_quatenary_branches == 0 else sum(map(len, quatenary_branches))/float(len(quatenary_branches))

        return quatenary_branches, no_of_quatenary_branches, round(avg_length_of_quatenary_branches, 1)

    def get_terminal_branches(self):
        terminal_branches = self.terminal_branches
        no_of_terminal_branches = len(terminal_branches)
        avg_length_of_terminal_branches = 0 if no_of_terminal_branches == 0 else sum(map(len, terminal_branches))/float(len(terminal_branches))

        return terminal_branches, no_of_terminal_branches, round(avg_length_of_terminal_branches, 1)

class Sholl:
    """
    Extract radius and no. of intersections for sholl analyses and other relevant features from the resulting sholl plot.
    """

    def __init__(self, cell_image, image_type, shell_step_size, polynomial_degree=3):

        """
        Args:

        cell_image: RGB cell image
        image_type: 'confocal' or 'DAB'
        shell_step_size: pixel difference between concentric circles for sholl analysis
        polynomial_degree (scalar): degree of polynomial for fitting regression model on sholl values

        """

        self.shell_step_size = shell_step_size
        self.polynomial_degree = polynomial_degree
        self.skeleton = Skeleton(cell_image, image_type)
        self.bounded_skeleton = self.skeleton.bounded_skeleton
        self.soma_on_bounded_skeleton = self.skeleton.soma_on_bounded_skeleton
        self.padded_skeleton = self.skeleton.padded_skeleton
        self.soma_on_padded_skeleton = self.skeleton.soma_on_padded_skeleton
        self.distances_from_soma = self.sholl_results()[0]
        self.no_of_intersections = self.sholl_results()[1]
        self.polynomial_model = self.polynomial_fit()
        self.determination_ratio()

    def concentric_coords_and_values(self):
        # concentric_coordinates: {radius values: [pixel coordinates on that radius]}
        # no_of_intersections: {radius values: no_of_intersection values}

        largest_radius = int(1.3*(np.max([self.soma_on_bounded_skeleton[1], abs(self.soma_on_bounded_skeleton[1]-self.bounded_skeleton.shape[1]),
            self.soma_on_bounded_skeleton[0], abs(self.soma_on_bounded_skeleton[0]-self.bounded_skeleton.shape[0])])))
        concentric_coordinates = defaultdict(list) # {100: [(10,10), ..] , 400: [(20,20), ..]}
        concentric_coordinates_intensities = defaultdict(list)
        concentric_radiuses = [radius for radius in range(self.shell_step_size, largest_radius, self.shell_step_size)]

        for (x, y), value in np.ndenumerate(self.padded_skeleton):
            for radius in concentric_radiuses:
                lhs = (x - self.soma_on_padded_skeleton[0])**2 + (y - self.soma_on_padded_skeleton[1])**2
                if abs((math.sqrt(lhs)-radius)) < 0.9:
                    concentric_coordinates[radius].append((x, y))
                    concentric_coordinates_intensities[radius].append(value)

        # array with intersection values corresponding to radii
        no_of_intersections = defaultdict()
        for radius, val in concentric_coordinates_intensities.items():
            intersec_indicies=[]
            indexes = [i for i, x in enumerate(val) if x]
            for index in indexes:
                intersec_indicies.append(concentric_coordinates[radius][index])
            img = np.zeros(self.padded_skeleton.shape)
            intersections = []
            for i, j in enumerate(intersec_indicies):
                img[j] = 1
            label_image = label(img)
            no_of_intersections[radius] = np.amax(label_image)

        return concentric_coordinates, no_of_intersections

    def sholl_results(self, plot=False):
        # return sholl radiuses and corresponding intersection values
        xs, ys = [], []
        concentric_coordinates, no_of_intersections = self.concentric_coords_and_values()
        for rad, val in no_of_intersections.items():
            xs.append(rad)
            ys.append(val)
        order = np.argsort(xs)

        if plot==True:
            astrocyte_skeleton_copy = copy.deepcopy(self.padded_skeleton)
            for radius, coordinates in concentric_coordinates.items():
                for coord in coordinates:
                    cell_image_with_circles = astrocyte_skeleton_copy
                    cell_image_with_circles[coord[0],coord[1]]=1

            # plot circles on skeleton
            fig, ax = plt.subplots(figsize=(10, 6))
            ax.imshow(cell_image_with_circles)
            # overlay soma on skeleton
            y, x = self.soma_on_padded_skeleton
            c = plt.Circle((x, y), 1, color='red')
            ax.add_patch(c)
            ax.set_axis_off()
            plt.tight_layout()
            plt.show()

            # plot sholl graph showing radius vs. no_of_intersections
            plt.plot(self.distances_from_soma, self.no_of_intersections)
            plt.xlabel("Distance from centre")
            plt.ylabel("No. of intersections")
            plt.show()

        return np.array(xs)[order], np.array(ys)[order]

    def polynomial_fit(self, plot=False):
        # Linear polynomial regression to describe the relationship between intersections vs. distance

        # till last non-zero value
        last_intersection_index = np.max(np.nonzero(self.no_of_intersections))
        self.non_zero_no_of_intersections = self.no_of_intersections[:last_intersection_index]
        self.non_zero_distances_from_soma = self.distances_from_soma[:last_intersection_index]

        y_data = self.non_zero_no_of_intersections
        reshaped_x = self.non_zero_distances_from_soma.reshape((-1, 1))

        x_ = preprocessing.PolynomialFeatures(degree=self.polynomial_degree, include_bias=False).fit_transform(reshaped_x)
        # create a linear regression model
        polynomial_model = linear_model.LinearRegression().fit(x_, y_data)

        self.polynomial_predicted_no_of_intersections = polynomial_model.predict(x_)

        if plot==True:
            # predict y from the data
            x_new = self.non_zero_distances_from_soma
            y_new = polynomial_model.predict(x_)
            # plot the results
            plt.figure(figsize=(4, 3))
            ax = plt.axes()
            ax.scatter(reshaped_x, y_data)
            ax.plot(x_new, y_new)
            ax.set_xlabel('x')
            ax.set_ylabel('y')
            ax.axis('tight')
            plt.show()

        return polynomial_model

    def enclosing_radius(self):
        # index of last non-zero value in the array containing radii
        return self.non_zero_distances_from_soma[len(self.non_zero_no_of_intersections) - (self.non_zero_no_of_intersections!=0)[::-1].argmax() - 1]

    def critical_radius(self):
        # radii_array[index of the max value in the array for no_of_intersections (polynomial plot)]
        return self.non_zero_distances_from_soma[np.argmax(self.polynomial_predicted_no_of_intersections)]

    def critical_value(self):
        # local maximum of the polynomial fit (Maximum no. of intersections)
        return round(np.max(self.polynomial_predicted_no_of_intersections), 2)

    def skewness(self):
        # indication of how symmetrical the polynomial distribution is around its mean
        x_ = preprocessing.PolynomialFeatures(degree=3, include_bias=False).fit_transform(self.non_zero_no_of_intersections.reshape((-1, 1)))
        return round(scipy.stats.skew(self.polynomial_model.predict(x_)), 2)

    def schoenen_ramification_index(self):
        # ratio between critical value and number of primary branches
        no_of_primary_branches = self.skeleton.get_primary_branches()[1]
        schoenen_ramification_index = self.critical_value()/no_of_primary_branches
        return round(schoenen_ramification_index, 2)

    def semi_log(self):
        # no. of intersections/circumference
        normalized_y = np.log(self.non_zero_no_of_intersections/(2*math.pi*self.non_zero_distances_from_soma))
        reshaped_x = self.non_zero_distances_from_soma.reshape((-1, 1))
        model = linear_model.LinearRegression().fit(reshaped_x, normalized_y)

        # predict y from the data
        x_new = self.non_zero_distances_from_soma
        y_new = model.predict(reshaped_x)
        r2 = model.score(reshaped_x, normalized_y)
        regression_intercept = model.intercept_
        regression_coefficient = -model.coef_[0]

        return r2, regression_intercept, regression_coefficient

    def log_log(self):
        # no. of intersections/circumference
        normalized_y = np.log(self.non_zero_no_of_intersections/(2*math.pi*self.non_zero_distances_from_soma))
        reshaped_x = self.non_zero_distances_from_soma.reshape((-1, 1))
        normalized_x = np.log(reshaped_x)
        model = linear_model.LinearRegression().fit(normalized_x, normalized_y)

        # predict y from the data
        x_new = normalized_x
        y_new = model.predict(normalized_x)
        r2 = model.score(normalized_x, normalized_y)
        regression_intercept = model.intercept_
        regression_coefficient = -model.coef_[0]

        return r2, regression_intercept, regression_coefficient

    def determination_ratio(self):
        semi_log_r2 = self.semi_log()[0]
        log_log_r2 = self.log_log()[0]
        determination_ratio = semi_log_r2/log_log_r2
        if determination_ratio>1:
            self.normalization_method="Semi-log"
        else:
            self.normalization_method="Log-log"

    def coefficient_of_determination(self):
        # how close the data are to the fitted regression (indicative of the level of explained variability in the data set)
        if self.normalization_method=="Semi-log":
            return round(self.semi_log()[0], 2)
        else:
            return round(self.log_log()[0], 2)

    def regression_intercept(self):
        # Y intercept of the logarithmic plot
        if self.normalization_method=="Semi-log":
            return round(self.semi_log()[1], 2)
        else:
            return round(self.log_log()[1], 2)

    def sholl_regression_coefficient(self):
        # rate of decay of no. of branches
        if self.normalization_method=="Semi-log":
            return round(self.semi_log()[2], 2)
        else:
            return round(self.log_log()[2], 2)

class analyze_cells:
    """
    Extract features of all the cells in different groups and implement PCA and group level sholl analysis.
    """

    def __init__(self, groups_folders, image_type, label, save_features=True, show_sholl_plot=True, shell_step_size=3):

        """
        Args:

        groups_folders (list): name of the input data folders corresponding to different subgroups
        image_type (string): 'confocal' or 'DAB'
        label (dictionary): group labels to be used for pca plots
        save_features (True/False): create text files containing feature values
        show_sholl_plot (True/False): show group sholl plot
        shell_step_size (scalar): difference between concentric circles

        """

        self.show_sholl_plot = show_sholl_plot
        self.image_type = image_type
        self.label = label
        self.shell_step_size = shell_step_size

        dataset = self.read_images(groups_folders)
        self.features = self.get_features(dataset)
        self.feature_names = ['surface_area', 'total_length', 'avg_process_thickness', 'convex_hull', 'no_of_forks', 'no_of_primary_branches', 'no_of_secondary_branches',
                                'no_of_tertiary_branches', 'no_of_quatenary_branches', 'no_of_terminal_branches', 'avg_length_of_primary_branches', 'avg_length_of_secondary_branches',
                                'avg_length_of_tertiary_branches', 'avg_length_of_quatenary_branches', 'avg_length_of_terminal_branches',
                                'critical_radius', 'critical_value', 'enclosing_radius', 'ramification_index', 'skewness', 'coefficient_of_determination',
                                'sholl_regression_coefficient', 'regression_intercept']
        if save_features==True:
            self.save_features()
        if show_sholl_plot==True:
            self.show_avg_sholl_plot(shell_step_size)

    def read_images(self, groups_folders):
        self.file_names=[]
        dataset=[]
        for group in groups_folders:
            group_data=[]
            for file in os.listdir(group):
                if not file.startswith('.'):
                    self.file_names.append((group+'/'+file))
                    image = io.imread(group+'/'+file)
                    group_data.append(image)
            dataset.append(group_data)

        return dataset

    def get_features(self, dataset):
        dataset_features=[]
        self.targets=[]

        if self.show_sholl_plot==True:
            self.sholl_original_plots=[]
            self.sholl_polynomial_plots=[]
            self.polynomial_models=[]

        self.group_counts=[]
        cell_count=0
        for group_no, group in enumerate(dataset):
            group_cell_count=0
            for cell_no, cell_image in enumerate(group):

                print(self.file_names[cell_count])

                cell_count+=1
                group_cell_count+=1

                self.targets.append(group_no)

                cell_features=[]

                astrocyte = Cell(cell_image, self.image_type)
                skeleton = Skeleton(cell_image, self.image_type)
                sholl = Sholl(cell_image, self.image_type, self.shell_step_size)

                cell_features.append(astrocyte.surface_area())
                cell_features.append(skeleton.total_length())
                cell_features.append(skeleton.avg_process_thickness())
                cell_features.append(skeleton.convex_hull())
                cell_features.append(skeleton.get_no_of_forks())
                cell_features.append(skeleton.get_primary_branches()[1])
                cell_features.append(skeleton.get_secondary_branches()[1])
                cell_features.append(skeleton.get_tertiary_branches()[1])
                cell_features.append(skeleton.get_quatenary_branches()[1])
                cell_features.append(skeleton.get_terminal_branches()[1])
                cell_features.append(skeleton.get_primary_branches()[2])
                cell_features.append(skeleton.get_secondary_branches()[2])
                cell_features.append(skeleton.get_tertiary_branches()[2])
                cell_features.append(skeleton.get_quatenary_branches()[2])
                cell_features.append(skeleton.get_terminal_branches()[2])
                cell_features.append(sholl.critical_radius())
                cell_features.append(sholl.critical_value())
                cell_features.append(sholl.enclosing_radius())
                cell_features.append(sholl.schoenen_ramification_index())
                cell_features.append(sholl.skewness())
                cell_features.append(sholl.coefficient_of_determination())
                cell_features.append(sholl.sholl_regression_coefficient())
                cell_features.append(sholl.regression_intercept())

                if self.show_sholl_plot==True:
                    self.sholl_original_plots.append((sholl.distances_from_soma, sholl.no_of_intersections))
                    self.sholl_polynomial_plots.append((sholl.non_zero_distances_from_soma, sholl.non_zero_no_of_intersections))
                    self.polynomial_models.append(sholl.polynomial_model)

                dataset_features.append(cell_features)

            self.group_counts.append(group_cell_count)
        return dataset_features

    def save_features(self):
        directory = os.getcwd()+'/Features'
        if os.path.exists(directory) and os.path.isdir(directory):
            shutil.rmtree(directory)
            os.mkdir(directory)
        else:
            os.mkdir(directory)

        def save_to_file(file_name, feature_name, feature_value):
            path = os.getcwd()+'/Features/'
            with open(path+feature_name+'.txt', 'a') as text_file:
                text_file.write("{} {} \n".format(file_name, feature_value))

        for cell_no, cell_features in enumerate(self.features):
            for feature_no, feature_val in enumerate(cell_features):
                save_to_file(self.file_names[cell_no], self.feature_names[feature_no], feature_val)

    def show_avg_sholl_plot(self, shell_step_size):
        original_plots_file = 'Original plots'
        polynomial_plots_file = 'Polynomial plots'

        directory = os.getcwd()+'/Sholl Results'

        if os.path.exists(directory) and os.path.isdir(directory):
            shutil.rmtree(directory)
            os.mkdir(directory)
        else:
            os.mkdir(directory)

        path = os.getcwd()+'/Sholl Results/'

        largest_radius = []
        no_of_intersections=[]

        with open(path+original_plots_file, 'w+') as text_file:
            for cell_no, plot in enumerate(self.sholl_original_plots):
                text_file.write("{} {} {} \n".format(self.file_names[cell_no], plot[0], plot[1]))

                # # get the max radius of each cell, as smallest and mid-level ones can be inferred from shell_step_size
                # largest_radius.append(max(plot[0]))
                # no_of_intersections.append(plot[1])

        with open(path+polynomial_plots_file, 'w+') as text_file:
            for cell_no, plot in enumerate(self.sholl_polynomial_plots):
                text_file.write("{} {} {} \n".format(self.file_names[cell_no], plot[0], plot[1]))

                # get the max radius of each cell, as smallest and mid-level ones can be inferred from shell_step_size
                largest_radius.append(max(plot[0]))
                no_of_intersections.append(plot[1])

        group_radiuses=[]
        sholl_intersections=[]
        for group_no, count in enumerate(self.group_counts):
            group_count = sum(self.group_counts[:group_no+1])
            group_radius = max(largest_radius[group_count-count:group_count])
            group_radiuses.append(group_radius)

            current_intersections = no_of_intersections[group_count-count:group_count]
            current_radiuses = range(shell_step_size, group_radius+1, shell_step_size)

            intersection_dict = defaultdict(list)
            for intersections in current_intersections:
                for i, intersection_val in enumerate(intersections):
                    intersection_dict[current_radiuses[i]].append(intersection_val)
            sholl_intersections.append(intersection_dict)

        with open(path+"Sholl values", 'w') as text_file:
            for group_no, group_sholl in enumerate(sholl_intersections):
                text_file.write("Group: {}\n".format(group_no))
                for radius, intersections in group_sholl.items():
                    text_file.write("{} {}\n".format(radius, intersections))

        for group_no, group_sholl in enumerate(sholl_intersections):
            x=[]
            y=[]
            e=[]
            for radius, intersections in group_sholl.items():
                x.append(radius)
                intersections = (intersections + self.group_counts[group_no] * [0])[:self.group_counts[group_no]]
                y.append(np.mean(intersections))
                e.append(scipy.stats.sem(intersections))
            plt.errorbar(x, y, yerr=e, label=self.label[group_no])

        plt.xlabel("Distance from soma")
        plt.ylabel("No. of intersections")
        plt.legend()
        plt.show()

    def pca(self, color_dict, marker):

        self.marker = marker

        def get_cov_ellipse(cov, centre, nstd, **kwargs):
            """
            Return a matplotlib Ellipse patch representing the covariance matrix
            cov centred at centre and scaled by the factor nstd.

            """

            # Find and sort eigenvalues and eigenvectors into descending order
            eigvals, eigvecs = np.linalg.eigh(cov)
            order = eigvals.argsort()[::-1]
            eigvals, eigvecs = eigvals[order], eigvecs[:, order]

            # The anti-clockwise angle to rotate our ellipse by
            vx, vy = eigvecs[:,0][0], eigvecs[:,0][1]
            theta = np.arctan2(vy, vx)

            # Width and height of ellipse to draw
            width, height = nstd * np.sqrt(eigvals)

            return Ellipse(xy=centre, width=width, height=height, angle=np.degrees(theta), **kwargs)

        pca_object = decomposition.PCA(2)

        # Scale data
        scaler = preprocessing.MaxAbsScaler()
        scaler.fit(self.features)
        X=scaler.transform(self.features)

        # fit on data
        pca_object.fit(X)

        # access values and vectors
        self.feature_significance = pca_object.components_

        # variance captured by principal components
        first_component_var = pca_object.explained_variance_ratio_[0]
        second_component_var = pca_object.explained_variance_ratio_[1]

        # transform data
        self.projected = pca_object.transform(X)

        first_component=self.projected[:,0]
        second_component=self.projected[:,1]

        with open("pca values", 'w') as text_file:
            text_file.write("First component:\n{}\nSecond component:\n{}".format(first_component, second_component))

        no_of_std = 3 # no. of standard deviations to show
        fig, ax = plt.subplots()
        fig.patch.set_facecolor('white')
        for l in np.unique(self.targets):
            ix = np.where(self.targets==l)
            first_component_mean = np.mean(first_component[ix])
            second_component_mean = np.mean(second_component[ix])
            cov = np.cov(first_component, second_component)
            ax.scatter(first_component[ix], second_component[ix], c=color_dict[l], s=40, label=self.label[l], marker=marker[l])
            e = get_cov_ellipse(cov, (first_component_mean, second_component_mean), no_of_std, fc=color_dict[l], alpha=0.4)
            ax.add_artist(e)

        plt.xlabel("PC 1 (Variance: %.1f%%)" % (first_component_var*100), fontsize=14)
        plt.ylabel("PC 2 (Variance: %.1f%%)" % (second_component_var*100), fontsize=14)
        plt.legend()
        plt.show()

    def plot_feature_histograms(self):
        fig, axes = plt.subplots(12, 2, figsize=(15, 12)) # 2 columns each containing 13 figures, total 22 features
        data = np.array(self.features)
        ko = data[np.where(np.array(self.targets) == 0)[0]] # define ko
        control = data[np.where(np.array(self.targets) == 1)[0]] # define control
        ax=axes.ravel() # flat axes with numpy ravel

        for i in range(len(self.feature_names)):
            _, bins=np.histogram(data[:,i], bins=40)
            ax[i].hist(ko[:,i], bins=bins, color='r',alpha=.5) # red color for malignant class
            ax[i].hist(control[:,i], bins=bins, color='g',alpha=0.3) # alpha is for transparency in the overlapped region
            ax[i].set_title(self.feature_names[i],fontsize=9)
            ax[i].axes.get_xaxis().set_visible(False) # the x-axis co-ordinates are not so useful, as we just want to look how well separated the histograms are
            ax[i].set_yticks(())

        ax[0].legend(self.marker, loc='best', fontsize=8)
        plt.tight_layout() # let's make good plots
        plt.show()

    def plot_feature_significance_heatmap(self):
        sorted_significance_order = np.flip(np.argsort(abs(self.feature_significance[0])))
        sorted_feature_significance = np.zeros(self.feature_significance.shape)
        sorted_feature_significance[0] = np.array(self.feature_significance[0])[sorted_significance_order]
        sorted_feature_significance[1] = np.array(self.feature_significance[1])[sorted_significance_order]
        sorted_feature_names = np.array(self.feature_names)[sorted_significance_order]

        plt.matshow(np.array(sorted_feature_significance), cmap='gist_heat')
        plt.yticks([0,1], ['1st Comp','2nd Comp'], fontsize=10)
        plt.colorbar()
        plt.xticks(range(len(sorted_feature_names)), sorted_feature_names, rotation=65, ha='left')
        plt.show()

    def plot_feature_significance_vectors(self):
        score = self.projected
        coeff = np.transpose(self.feature_significance)
        labels=self.feature_names
        xs = score[:,0]
        ys = score[:,1]
        n = coeff.shape[0]
        scalex = 1.0/(xs.max() - xs.min())
        scaley = 1.0/(ys.max() - ys.min())

        plt.figure(figsize=(10, 9))
        ax = plt.axes()
        ax.scatter(xs * scalex,ys * scaley, c = self.targets)
        for i in range(n):
            ax.arrow(0, 0, coeff[i,0], coeff[i,1],color = 'r',alpha = 0.5)
            if labels is None:
                ax.text(coeff[i,0]* 1.15, coeff[i,1] * 1.15, "Var"+str(i+1), color = 'g', ha = 'center', va = 'center')
            else:
                ax.text(coeff[i,0]* 1.15, coeff[i,1] * 1.15, labels[i], color = 'g', ha = 'center', va = 'center')
        ax.set_xlabel("PC {}".format(1))
        ax.set_ylabel("PC {}".format(2))
        plt.show()


# Import `Site` and the `ui` module from the `h2o_wave` package
from h2o_wave import site, ui,main
import time
import requests

def mifun(text):
    if len(text)<1:
        return text
    else:
        response = requests.post('http://api/ner', data = {'text':'Each component, API-Service'})
        #print(response)
        return response
# Get the web page at route '/demo'.
# If you're running this example on your local machine,
# this page will refer to http://localhost:10101/demo.
page = site['/ner_ui']

# Add a Markdown card named `hello` to the page.
# page['hello'] = ui.markdown_card(
#     box='1 1 2 2',
#     title='Hello World!',
#     content='And now for something completely different!',
# )
text='asdf'
page['ner'] = ui.form_card(box='1 1 4 10', items=[
        ui.textbox(name='text', label='English', value=text or '', multiline=True),
        ui.label('Pig Latin'),
        ui.text(mifun(text) or '*Type in some text above to translate to Pig Latin!*'),
    ])

beer_verse = '''={{before}} bottles of beer on the wall, {{before}} bottles of beer.

Take one down, pass it around, {{after}} bottles of beer on the wall...
'''

beer_card = page.add('wall', ui.markdown_card(
    box='6 1 4 2',
    title='99 Bottles of Beer',
    content=beer_verse,
    data=dict(before='99', after='98'),
))

for i in range(99, 0, -1):
    beer_card.data.before = str(i)
    beer_card.data.after = str(i - 1)
    page.save()
    time.sleep(1)


# -*- encoding: utf-8 -*-

import base64

import time
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta

from openerp.osv import fields, osv
from openerp.tools.translate import _
from openerp import tools

import logging

# _logger = logging.getLogger(__name__)

class account_rus_import(osv.osv_memory):
    _name = 'account.rus.import'
    _description = 'Import Russian statements file'
    _columns = {
        'file_data': fields.binary('Import statements File', required=True),
        'file_fname': fields.char('Import statements Filename', size=128, required=True),
        'note': fields.text('Log'),
        'temporary_account_id': fields.many2one('account.account', 'Temporary Account', help="It acts as a temporary account for general amount", required=True),  # domain="[('type','!=','view')]"
    }

    def _get_default_tmp_account(self, cr, uid, context):
        tmp_accounts = self.pool.get('account.account').search(cr, uid, [('code', '=', '490000')])
        if tmp_accounts and len(tmp_accounts) > 0:
            tmp_account_id = tmp_accounts[0]
        else:
            tmp_account_id = False
        return tmp_account_id

    _defaults = {
        'file_fname': lambda *a: '',
        'temporary_account_id': _get_default_tmp_account,
    }

    def file_parsing(self, cr, uid, ids, context=None, batch=False, rfile=None, rfilename=None):

        def acc_number_parsing(obj, line_number):
            acc_res = {}
            acc_res_number = ''
            while rmspaces(obj[line_number][0]) != u'КонецРасчСчет':  # !!! Документ
                if obj[line_number][0] == u'ДатаНачала':
                    acc_res['begin_date'] = time.strftime(tools.DEFAULT_SERVER_DATE_FORMAT,
                                                          time.strptime(rmspaces(obj[line_number][1]), '%d.%m.%Y'))
                elif obj[line_number][0] == u'ДатаКонца':
                    acc_res['end_date'] = time.strftime(tools.DEFAULT_SERVER_DATE_FORMAT,
                                                        time.strptime(rmspaces(obj[line_number][1]), '%d.%m.%Y'))
                elif obj[line_number][0] == u'РасчСчет':
                    acc_res_number = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'НачальныйОстаток':
                    acc_res['balance_start'] = float(rmspaces(obj[line_number][1]))
                elif obj[line_number][0] == u'ВсегоПоступило':
                    acc_res['balance_plus'] = float(rmspaces(obj[line_number][1]))
                elif obj[line_number][0] == u'ВсегоСписано':
                    acc_res['balance_minus'] = float(rmspaces(obj[line_number][1]))
                elif obj[line_number][0] == u'КонечныйОстаток':
                    acc_res['balance_end'] = float(rmspaces(obj[line_number][1]))
                line_number += 1
            line_number += 1
            return acc_res_number, acc_res, line_number

        if context is None:
            context = {}
        if batch:
            rfile = str(rfile)
            rfilename = rfilename
        else:
            data = self.browse(cr, uid, ids)[0]
            try:
                rfile = data.file_data
                rfilename = data.file_fname
                temporaryaccount = data.temporary_account_id.id
            except:
                raise osv.except_osv(_('Error'), _('Wizard in incorrect state. Please hit the Cancel button'))
                # return {}
        recordlist = unicode(base64.decodestring(rfile), 'windows-1251', 'strict').split('\n')
        strobj = []
        for line in recordlist:
            strobj.append(line.split('='))

        format_import_file = ''
        encoding_file = ''
        statements = {}
        note = []
        inc_desc = 1
        if rmspaces(recordlist[0]) != '1CClientBankExchange':
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))

        if strobj[inc_desc][0] == u'ВерсияФормата':
            format_import_file = rmspaces(strobj[inc_desc][1])
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1
        else:
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))

        if strobj[inc_desc][0] == u'Кодировка':
            encoding_file = rmspaces(strobj[inc_desc][1])
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1
        else:
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))

        if strobj[inc_desc][0] == u'Отправитель':
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1
        if strobj[inc_desc][0] == u'Получатель':
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1
        else:
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))

        if strobj[inc_desc][0] == u'ДатаСоздания':
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1
        if strobj[inc_desc][0] == u'ВремяСоздания':
            note.append(recordlist[inc_desc] + '\n')
            inc_desc += 1

        if strobj[inc_desc][0] == u'ДатаНачала':
            note.append(recordlist[inc_desc] + '\n')
            statements['begin_date'] = time.strftime(tools.DEFAULT_SERVER_DATE_FORMAT,
                                                     time.strptime(rmspaces(strobj[inc_desc][1]), '%d.%m.%Y'))
            tmp_acc, tmp_acc_res, slide_i = acc_number_parsing(strobj, inc_desc)
            inc_desc += 1
        else:
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))
        if strobj[inc_desc][0] == u'ДатаКонца':
            note.append(recordlist[inc_desc] + '\n')
            statements['end_date'] = time.strftime(tools.DEFAULT_SERVER_DATE_FORMAT,
                                                   time.strptime(rmspaces(strobj[inc_desc][1]), '%d.%m.%Y'))
            inc_desc += 1
        else:
            raise osv.except_osv(_('Error'), _('Incorrect description of import file'))
        acc_numbers = []
        while strobj[inc_desc][0] == u'РасчСчет':
            acc_number = {}
            acc_number['detail'] = []
            acc_number['statement_line'] = []
            acc_number['acc_number'] = rmspaces(strobj[inc_desc][1])
            acc_number['journal_id'] = False
            acc_number['bank_account'] = False
            bank_ids = self.pool.get('res.partner.bank').search(cr, uid,
                                                                [('acc_number', '=', acc_number['acc_number'])])
            if bank_ids and len(bank_ids) > 0:
                bank_accs = self.pool.get('res.partner.bank').browse(cr, uid, bank_ids)
                for bank_acc in bank_accs:
                    if bank_acc.journal_id.id:
                        acc_number['journal_id'] = bank_acc.journal_id
                        acc_number['bank_account'] = bank_acc
                        break
            if not acc_number['bank_account']:
                raise osv.except_osv(_('Error'), _("No matching Bank Account (with Account Journal) found.\n\n"
                                                   "Please set-up a Bank Account with as Account Number '%s' "
                                                   "and an Account Journal.") % (acc_number['acc_number']))
            acc_numbers.append(acc_number)
            inc_desc += 1

        def statement_line_parsing(obj, line_number):
            statementLine = {}
            statementLine['note'] = []
            while rmspaces(obj[line_number][0]) != u'КонецДокумента':
                if obj[line_number][0] == u'Номер':
                    statementLine['ref'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'Дата':
                    statementLine['date'] = time.strftime(tools.DEFAULT_SERVER_DATE_FORMAT, time.strptime(rmspaces(obj[line_number][1]),'%d.%m.%Y'))
                elif obj[line_number][0] == u'Сумма':
                    statementLine['amount'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'ПлательщикСчет':
                    statementLine['payer_acc'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'Плательщик':
                    statementLine['payer'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'ПлательщикИНН':
                    statementLine['payer_inn'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'ПолучательСчет':
                    statementLine['recipient_acc'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'Получатель':
                    statementLine['recipient'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'ПолучательИНН':
                    statementLine['recipient_inn'] = rmspaces(obj[line_number][1])
                elif obj[line_number][0] == u'НазначениеПлатежа':
                    statementLine['name'] = rmspaces(obj[line_number][1])
                else:
                    statementLine['note'].append(obj[line_number][0] + ': ' + obj[line_number][1])
                line_number += 1
            line_number += 1
            return statementLine, line_number

        #for i in range(inc_desc,len(strobj)-1):
        i = inc_desc
        while rmspaces(strobj[i][0]) != u'КонецФайла':
            # _logger = logging.getLogger(__name__)
            # _logger.info('ENDFILE %s %s %s', i, len(strobj)-1, strobj[i][0])
            if strobj[i][0] == u'СекцияРасчСчет':  # Документ
                # _logger = logging.getLogger(__name__)
                # _logger.info('SEC RASH')
                for acc_one in acc_numbers:
                    if acc_one['acc_number'] == tmp_acc:
                        acc_one['detail'].append(tmp_acc_res)
                        break

            elif strobj[i][0] == u'СекцияДокумент':
                tmp_statementLine, slide_i = statement_line_parsing(strobj, i)
                # _logger = logging.getLogger(__name__)
                # _logger.info('SEC DOC')
                for acc_one in acc_numbers:
                    if acc_one['acc_number'] == tmp_statementLine['payer_acc'] or \
                                    acc_one['acc_number'] == tmp_statementLine['recipient_acc']:
                        tmp_statementLine['sequence'] = len(acc_one['statement_line']) + 1
                        acc_one['statement_line'].append(tmp_statementLine)
                        break
            if i < (len(strobj)-2):
                i += 1

        for slide_i, statement in enumerate(acc_numbers):
            period_id = self.pool.get('account.period').search(cr, uid,
                                                        [('company_id', '=', statement['journal_id'].company_id.id),
                                                        ('date_start', '<=', statements['end_date']),
                                                        ('date_stop', '>=', statements['end_date'])])
            if not period_id and len(period_id) == 0:
                raise osv.except_osv(_('Error'), _("The Statement New Balance date doesn't fall within a defined Accounting Period! Please create the Accounting Period for date %s for the company %s.") % (statements['end_date'], statement['journal_id'].company_id.name))
            statement['period_id'] = period_id[0]

            statement['note'] = note
            cr.execute('SELECT balance_end_real \
            FROM account_bank_statement \
            WHERE journal_id = %s and date <= %s \
            ORDER BY date DESC,id DESC LIMIT 1', (statement['journal_id'].id, statements['begin_date']))
            res = cr.fetchone()
            balance_start_check = res and res[0]
            if balance_start_check == None:
                if statement['journal_id'].default_debit_account_id and (statement['journal_id'].default_credit_account_id == statement['journal_id'].default_debit_account_id):
                    balance_start_check = statement['journal_id'].default_debit_account_id.balance
                else:
                    raise osv.except_osv(_('Error'), _("Configuration Error in journal %s!\nPlease verify the Default Debit and Credit Account settings.") % statement['journal_id'].name)
            if balance_start_check != tmp_acc_res['balance_start']:  # statement['balance_start']
                statement['note'].append(_("The Statement %s Starting Balance (%.2f) does not correspond with the previous Closing Balance (%.2f) in journal %s!") % (statement['acc_number'] + ' #' + statements['begin_date'] + ':' + statements['end_date'], tmp_acc_res['balance_start'], balance_start_check, statement['journal_id'].name))
            if not(statement.get('period_id')):
                raise osv.except_osv(_('Error'), _(' No transactions or no period in file !'))
            data = {
                'name': statement['acc_number'] + ' #' + statements['begin_date'] + ':' + statements['end_date'],
                'date': datetime.now(),
                'journal_id': statement['journal_id'].id,
                'period_id': statement['period_id'],
                'balance_start': tmp_acc_res['balance_start'],   # 'balance_start': statement['balance_start'],
                'balance_end_real': tmp_acc_res['balance_end'],
            }
            statement['id'] = self.pool.get('account.bank.statement').create(cr, uid, data, context=context)
            for line in statement['statement_line']:
                partner = None
                partner_id = None
                invoice = False
                if line['payer_acc'] == statement['acc_number']:
                    pass
                else:
                    ids = self.pool.get('res.partner.bank').search(cr, uid, [('acc_number', '=', str(line['payer_acc']))])
                    _logger = logging.getLogger(__name__)
                    _logger.info('IDS %s', ids)
                    p_id = None

                    if ids and len(ids) > 0:
                        partner = self.pool.get('res.partner.bank').browse(cr, uid, ids[0], context=context)
                        _logger = logging.getLogger(__name__)
                        _logger.info('PARTNER %s', partner)
                        p_id = partner.partner_id.id
                        line['account'] = partner.partner_id.property_account_receivable.id
                        ids = ids[0]
                    else:
                        _logger = logging.getLogger(__name__)
                        _logger.info('PARTNER_NO %s', ids)
                        ids = ''
                    if not partner and not invoice:
                        line['account'] = temporaryaccount
                        line['name'] = line['name'] + '\n' + line['payer_inn']
                    data = {
                        'name': line['name'],
                        'note': '\n'.join(line['note']),
                        'date': line['date'],
                        'amount': line['amount'],
                        'partner_id': p_id,
                        #'account_id': line['account'], #Looks like reconsilation will not work with it
                        'statement_id': statement['id'],
                        'ref': line['ref'],
                        'sequence': line['sequence'],
                        'bank_account_id': ids,
                    }
                    data_check = {
                        'date': line['date'],
                        'amount': line['amount'],
                        'ref': line['ref'],
                        'account_id': line['account'],
                    }
                    ids_line_statement = self.pool.get('account.bank.statement.line').search(cr, uid,
                                                                                [('date', '=', data_check['date']),
                                                                                ('amount', '=', data_check['amount']),
                                                                                ('ref', '=', data_check['ref']),
                                                                                #('account_id', '=', data_check['account_id'])
                                                                                ])
                    if ids_line_statement:
                        statement['note'].append(_('Statement line %s from %s alredy exist.') %
                                                (data_check['ref'], data_check['date']))
                    else:
                        self.pool.get('account.bank.statement.line').create(cr, uid, data, context=context)

                if statement['note']:
                    self.pool.get('account.bank.statement').write(cr, uid, [statement['id']],
                                                              {'rusimport_note': '\n'.join(statement['note'])},
                                                              context=context)
        model, action_id = self.pool.get('ir.model.data').get_object_reference(cr, uid,
                                                                               'account', 'action_bank_statement_tree')
        action = self.pool[model].browse(cr, uid, action_id, context=context)
        return {
            'name': action.name,
            'view_type': action.view_type,
            'view_mode': action.view_mode,
            'res_model': action.res_model,
            'domain': action.domain,
            'context': action.context,
            'type': 'ir.actions.act_window',
            'search_view_id': action.search_view_id.id,
            'views': [(v.view_id.id, v.view_mode) for v in action.view_ids]
        }

def rmspaces(s):
    return " ".join(s.split())


from django.shortcuts import render
from django.shortcuts import get_object_or_404
from store.models import *
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse,HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.db.models import Avg

# Create your views here.

def index(request):
    return render(request, 'store/index.html')

def bookDetailView(request, bid):
    template_name = 'store/book_detail.html'
    try:
        book = Book.objects.get(pk=bid)
    except:
        return HttpResponse('
Requested Book Not Available!
')

    num_available = len(BookCopy.objects.filter(book__exact=book,status__exact=True))

    context = {
        'book': book, # set this to an instance of the required book
        'num_available': num_available, # set this to the number of copies of the book available, or 0 if the book isn't available
    }
    # START YOUR CODE HERE

    return render(request, template_name, context=context)

@csrf_exempt
def bookListView(request):
    template_name = 'store/book_list.html'
    get_data = request.GET

    books = Book.objects.filter(
        title__icontains=get_data.get('title',''),
        author__icontains=get_data.get('author',''),
        genre__icontains=get_data.get('genre', '')
    )
    context = {
        'books': books, # set this to the list of required books upon filtering using the GET parameters
                       # (i.e. the book search feature will also be implemented in this view)
    }
    return render(request, template_name, context=context)

@login_required
def viewLoanedBooks(request):
    template_name = 'store/loaned_books.html'
    books = BookCopy.objects.filter(borrower__exact=request.user)
    context = {
        'books': books,
    }
    '''
    The above key 'books' in the context dictionary should contain a list of instances of the
    BookCopy model. Only those book copies should be included which have been loaned by the user.
    '''
    # START YOUR CODE HERE

    return render(request, template_name, context=context)

@csrf_exempt
@login_required
def loanBookView(request):

    '''
    Check if an instance of the asked book is available.
    If yes, then set the message to 'success', otherwise 'failure'
    '''
    # START YOUR CODE HERE
    if not request.user.is_authenticated:
        return render(request,'login.html',{"message":"Login to loan book"})

    book_id = request.POST['bid']  # get the book id from post data
    books = BookCopy.objects.filter(book_id__exact=book_id, status__exact=True)
    if books:
            books[0].borrower = request.user
            books[0].borrow_date = datetime.date.today()
            books[0].status = False
            books[0].save()
            message = "success"
    else:
        message = "failure"
        response_data = {
            'message': message,
        }

    return JsonResponse(response_data)

'''
FILL IN THE BELOW VIEW BY YOURSELF.
This view will return the issued book.
You need to accept the book id as argument from a post request.
You additionally need to complete the returnBook function in the loaned_books.html file
to make this feature complete
'''
@csrf_exempt
@login_required
def returnBookView(request):
    if request.method == "POST":
        try:
            book_id = request.POST['bid']
            book = BookCopy.objects.get(pk=book_id)
            book.borrower = None
            book.borrow_date = None
            book.status = True
            book.save()
            return JsonResponse( {"message":"Book returned successfully"} )
        except:
            return JsonResponse( {"message":"Book not found"} )
    else:
        return JsonResponse( {"message":"Invalid request method"} )

@csrf_exempt
@login_required
def rateBookView(request):
     if request.method == "POST":
         book_id = request.POST['bid']
         username = request.user.username
         new_rating = request.POST['rating']
         book = Book.objects.get(pk=book_id)

         if new_rating>10 or new_rating<0:
             return JsonResponse({'message':'Rating not in range 0-10'})
         else:
             try:
                 current_user_book_rating_object = BookRating.objects.get_or_create(book=book, username=username)
                 current_user_book_rating_object.rating = new_rating
                 current_user_book_rating_object.save()

                 book.rating = BookRating.objects.filter(book = book).aggregate(rating=Avg('rating'))['rating']
                 book.save()
                 return JsonResponse({'message':'success'})
             except:
                 return JsonResponse({'message':"error"})

     else:
        return JsonResponse({'message':"Invalid request method"})


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
import re
import time

from .base import *  # noqa

# don't use an unicode string
localeID = 'ja_JP'
dateSep = ['-']
timeSep = [':', 'h']
meridian = ['午前', '午後']
usesMeridian = True
uses24 = True
WeekdayOffsets = {}
MonthOffsets = {}
useAccuracy = False
tryRegexParsing = True

# always lowercase any lookup values - helper code expects that
Weekdays = [
    '月曜日', '火曜日', '水曜日', '木曜日', '金曜日', '土曜日', '日曜日',
]

shortWeekdays = [
    '月', '火', '水', '木', '金', '土', '日',
]

Months = [
    '1月', '2月', '3月', '4月', '5月', '6月',
    '7月', '8月', '9月', '10月', '11月', '12月',
]

# We do not list 'mar' as a short name for 'mars' as it conflicts with
# the 'mar' of 'mardi'
shortMonths = [
    '1月', '2月', '3月', '4月', '5月', '6月',
    '7月', '8月', '9月', '10月', '11月', '12月',
]

# use the same formats as ICU by default
dateFormats = {
    'full': 'yyyy年MMMM月d日のEEEE',
    'long': 'yyyy MMMM d',
    'medium': 'yyyy MMM d',
    'short': 'yy-M-d'
}

timeFormats = {
    'full': 'hh:mm:ss a z',
    'long': 'h:mm:ss a z',
    'medium': 'h:mm:ss a',
    'short': 'h:mm a',
}

dp_order = ['y', 'm', 'd']

# Used to parse expressions like "in 5 hours"
numbers = {
    '零': 0,
    '一': 1,
    '二': 2,
    '三': 3,
    '四': 4,
    '五': 5,
    '六': 6,
    '七': 7,
    '八': 8,
    '九': 9,
    '十': 10,
}

decimal_mark = '.'

# this will be added to re_values later
units = {
    'seconds': ['秒'],
    'minutes': ['分'],
    'hours': ['時'],
    'days': ['日'],
    'weeks': ['週'],
    'months': ['月'],
    'years': ['年'],
}

# text constants to be used by later regular expressions
re_values = {
    'specials': '',
    'timeseparator': ":",
    'rangeseparator': '-',
    'daysuffix': '日',
    'meridian': '午前|夜明け|午後|昼|夜',
    'qunits': '時|分|秒|日|週|月|年',
    'now': ['今'],
}

# Used to adjust the returned date before/after the source
Modifiers = {
    '前々': -2,
    '先々': -2,
    '先': -1,
    '前': -1,
    '過去': -1,
    '次の': 1,
    '次': 1,
    '明後': 2,
    '後': 1,
    '再来': 2,
    '来': 1,
}

dayOffsets = {
    '明後日': 2,
    'あさって': 2,
    '明日': 1,
    '今日': 0,
    '昨日': -1,
    '前日': -1,
    '一昨日': -2,
    'おととい': -2,
}

# special day and/or times, i.e. lunch, noon, evening
# each element in the dictionary is a dictionary that is used
# to fill in any value to be replace - the current date/time will
# already have been populated by the method buildSources
re_sources = {
    '朝': {'hr': 6, 'mn': 0, 'sec': 0},
    '午前': {'hr': 9, 'mn': 0, 'sec': 0},
    '正午': {'hr': 12, 'mn': 0, 'sec': 0},
    '午後': {'hr': 13, 'mn': 0, 'sec': 0},
    '昼': {'hr': 13, 'mn': 0, 'sec': 0},
    '夜': {'hr': 19, 'mn': 0, 'sec': 0},
}

small = {
    '零': 0,
    '一': 1,
    '二': 2,
    '三': 3,
    '四': 4,
    '五': 5,
    '六': 6,
    '七': 7,
    '八': 8,
    '九': 9,
    '十': 10,
}

magnitude = {
    '百': 100,
    '千': 1000,
    '万': 10000,
    '億': 100000000,
    '兆': 1000000000000,
    '京': 10000000000000000,
}

ignore = (',', 'の')
cre_days = re.compile(rf'''\b((?P<qty>\d+)|(?P<nums>{'|'.join(numbers)}))\s*日\s*(?P<backforth>前|後)\b''', re.IGNORECASE)
cre_weeks = re.compile(rf'''\b\s*((?P<modifiers>{'|'.join(Modifiers)})|(?P<nums>{'|'.join(numbers)}))\s*週の?(?P<weekday>{'|'.join(Weekdays)})\b''', re.IGNORECASE)
cre_date1 = re.compile(r'((?P<year>(\d{2})?(\d{2}))-)?(?P<month>1[0-2]|0?[1-9])-(?P<day>3[01]|[12][0-9]|0?[1-9])')
cre_date2 = re.compile(r'((?P<year>(\d{2})?(\d{2}))\/)?(?P<month>1[0-2]|0?[1-9])\/(?P<day>3[01]|[12][0-9]|0?[1-9])')
cre_date3 = re.compile(r'(((?P<year>(\d{2})?(\d{2}))\s*年)?\s*(?P<month>1[0-2]|0?[1-9])\s*月)?\s*(?P<day>3[01]|[12][0-9]|0?[1-9])\s*日')

def parseRegex(s, sourceTime):
    m = cre_days.match(s)
    if m:
        qty = m.group('qty')
        nums = m.group('nums')
        backforth = m.group('backforth') == '後'

        if qty:
            n = int(qty)
        elif nums:
            n = numbers[nums]

        multiplier = 1 if backforth else -1
        day_delta = n * multiplier
        dt = datetime.datetime.fromtimestamp(time.mktime(sourceTime)) + datetime.timedelta(days=day_delta)
        sourceTime = dt.timetuple()
        return sourceTime, True

    m = cre_weeks.match(s)
    if m:
        modifiers = m.group('modifiers')
        nums = m.group('nums')
        weekday = m.group('weekday')

        if modifiers:
            n = Modifiers[modifiers]
        elif nums:
            n = numbers[nums]

        if weekday:
            w = -sourceTime.tm_wday + Weekdays.index(weekday)
        else:
            w = 0

        day_delta = n*7 + w
        dt = datetime.datetime.fromtimestamp(time.mktime(sourceTime)) + datetime.timedelta(days=day_delta)
        sourceTime = dt.timetuple()
        return sourceTime, True

    m = cre_date1.match(s)
    if not m:
        m = cre_date2.match(s)
    if not m:
        m = cre_date3.match(s)

    if m:
        year_str = m.group('year')
        month_str = m.group('month')
        day_str = m.group('day')

        year = int(year_str) if year_str else sourceTime.tm_year
        if year < 100:
            year += 2000
        month = int(month_str) if month_str else sourceTime.tm_mon
        day = int(day_str) if day_str else sourceTime.tm_mday

        return datetime.datetime(year, month, day).timetuple(), True

    return sourceTime, False


# Copyright 2016-2020 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from builtins import str
from collections import OrderedDict
from importlib import import_module
import os.path
import sys
import unittest

from tests.common_functions import create_abstract_model, add_components_and_load_data

TEST_DATA_DIRECTORY = os.path.join(
    os.path.dirname(__file__), "..", "..", "..", "test_data"
)

# Import prerequisite modules
PREREQUISITE_MODULE_NAMES = [
    "temporal.operations.timepoints",
    "temporal.operations.horizons",
    "temporal.investment.periods",
    "geography.load_zones",
    "project",
]
NAME_OF_MODULE_BEING_TESTED = "project.capacity.capacity_types.gen_ret_bin"
IMPORTED_PREREQ_MODULES = list()
for mdl in PREREQUISITE_MODULE_NAMES:
    try:
        imported_module = import_module("." + str(mdl), package="gridpath")
        IMPORTED_PREREQ_MODULES.append(imported_module)
    except ImportError:
        print("ERROR! Module " + str(mdl) + " not found.")
        sys.exit(1)
# Import the module we'll test
try:
    MODULE_BEING_TESTED = import_module(
        "." + NAME_OF_MODULE_BEING_TESTED, package="gridpath"
    )
except ImportError:
    print("ERROR! Couldn't import module " + NAME_OF_MODULE_BEING_TESTED + " to test.")

class TestGenRetBin(unittest.TestCase):
    """ """

    def test_add_model_components(self):
        """
        Test that there are no errors when adding model components
        :return:
        """
        create_abstract_model(
            prereq_modules=IMPORTED_PREREQ_MODULES,
            module_to_test=MODULE_BEING_TESTED,
            test_data_dir=TEST_DATA_DIRECTORY,
            subproblem="",
            stage="",
        )

    def test_load_model_data(self):
        """
        Test that data are loaded with no errors
        :return:
        """
        add_components_and_load_data(
            prereq_modules=IMPORTED_PREREQ_MODULES,
            module_to_test=MODULE_BEING_TESTED,
            test_data_dir=TEST_DATA_DIRECTORY,
            subproblem="",
            stage="",
        )

    def test_data_loaded_correctly(self):
        """
        Test that the data loaded are as expected
        :return:
        """
        m, data = add_components_and_load_data(
            prereq_modules=IMPORTED_PREREQ_MODULES,
            module_to_test=MODULE_BEING_TESTED,
            test_data_dir=TEST_DATA_DIRECTORY,
            subproblem="",
            stage="",
        )
        instance = m.create_instance(data)

        # Set: GEN_RET_BIN_OPR_PRDS
        expected_gen_set = [("Clunky_Old_Gen2", 2020), ("Clunky_Old_Gen2", 2030)]
        actual_gen_set = sorted([(g, p) for (g, p) in instance.GEN_RET_BIN_OPR_PRDS])
        self.assertListEqual(expected_gen_set, actual_gen_set)

        # Param: gen_ret_bin_capacity_mw
        expected_cap = {("Clunky_Old_Gen2", 2020): 10, ("Clunky_Old_Gen2", 2030): 10}
        actual_cap = {
            (g, p): instance.gen_ret_bin_capacity_mw[g, p]
            for (g, p) in instance.GEN_RET_BIN_OPR_PRDS
        }
        self.assertDictEqual(expected_cap, actual_cap)

        # Set: GEN_RET_BIN
        expected_gens = ["Clunky_Old_Gen2"]
        actual_gens = [g for g in instance.GEN_RET_BIN]
        self.assertListEqual(expected_gens, actual_gens)

        # Set: OPR_PRDS_BY_GEN_RET_BIN
        expected_periods_by_generator = {"Clunky_Old_Gen2": [2020, 2030]}
        actual_periods_by_generator = {
            g: [p for p in instance.OPR_PRDS_BY_GEN_RET_BIN[g]]
            for g in instance.GEN_RET_BIN
        }
        self.assertDictEqual(expected_periods_by_generator, actual_periods_by_generator)

        # Param: gen_ret_bin_first_period
        expected_first_period = {"Clunky_Old_Gen2": 2020}
        actual_first_period = {
            g: instance.gen_ret_bin_first_period[g] for g in instance.GEN_RET_BIN
        }
        self.assertDictEqual(expected_first_period, actual_first_period)

        # Param: gen_ret_bin_capacity_mw
        expected_cap = {("Clunky_Old_Gen2", 2020): 10, ("Clunky_Old_Gen2", 2030): 10}
        actual_cap = {
            (g, p): instance.gen_ret_bin_capacity_mw[g, p]
            for (g, p) in instance.GEN_RET_BIN_OPR_PRDS
        }
        self.assertDictEqual(expected_cap, actual_cap)

        # Param: gen_ret_bin_fixed_cost_per_mw_yr
        expected_cost = {
            ("Clunky_Old_Gen2", 2020): 1000,
            ("Clunky_Old_Gen2", 2030): 1000,
        }
        actual_cost = {
            (g, p): instance.gen_ret_bin_fixed_cost_per_mw_yr[g, p]
            for (g, p) in instance.GEN_RET_BIN_OPR_PRDS
        }
        self.assertDictEqual(expected_cost, actual_cost)

if __name__ == "__main__":
    unittest.main()


import logging
import json
import os
import signal
import sys
import urllib
import urlparse

from multiprocessing.process import Process

sys.path.insert(0, os.path.abspath(os.path.realpath(__file__) + '/../../../'))

from oauth2 import Provider
from oauth2.grant import ClientCredentialsGrant
from oauth2.tokengenerator import Uuid4
from oauth2.store.memory import ClientStore, TokenStore
from oauth2.web.tornado import OAuth2Handler
from tornado.ioloop import IOLoop
from tornado.web import Application, url
import oauth2
from tdaemon.handler import APIHandler
from tdaemon.libs.utils import MongoDB
import tdaemon.libs.utils as utils

class Oauth2libHandler(APIHandler):
    def get(self):
        try:
            client_store = ClientStore()
            client_store.add_client(client_id="abc", client_secret="xyz",
                                    redirect_uris=[],
                                    authorized_grants=[oauth2.grant.ClientCredentialsGrant.grant_type])

            token_store = TokenStore()

            # Generator of tokens
            token_generator = oauth2.tokengenerator.Uuid4()
            token_generator.expires_in[oauth2.grant.ClientCredentialsGrant.grant_type] = 3600

            provider = Provider(access_token_store=token_store,
                                auth_code_store=token_store, client_store=client_store,
                                token_generator=token_generator)
            # provider.add_grant(AuthorizationCodeGrant(site_adapter=TestSiteAdapter()))
            provider.add_grant(ClientCredentialsGrant())
        except Exception as e:
            result = {"success":0,"return_code":unicode(e),"error_msg":utils.format_error()}

        self.finish(result)

def run_auth_server():
    client_store = ClientStore()
    client_store.add_client(client_id="abc", client_secret="xyz",
                            redirect_uris=[],
                            authorized_grants=[oauth2.grant.ClientCredentialsGrant.grant_type])

    token_store = TokenStore()

    # Generator of tokens
    token_generator = oauth2.tokengenerator.Uuid4()
    token_generator.expires_in[oauth2.grant.ClientCredentialsGrant.grant_type] = 3600

    provider = Provider(access_token_store=token_store,
                        auth_code_store=token_store, client_store=client_store,
                        token_generator=token_generator)
    # provider.add_grant(AuthorizationCodeGrant(site_adapter=TestSiteAdapter()))
    provider.add_grant(ClientCredentialsGrant())

    try:
        app = Application([
            url(provider.authorize_path, OAuth2Handler, dict(provider=provider)),
            url(provider.token_path, OAuth2Handler, dict(provider=provider)),
        ])

        app.listen(8080)
        print("Starting OAuth2 server on http://localhost:8080/...")
        IOLoop.current().start()

    except KeyboardInterrupt:
        IOLoop.close()

def main():
    auth_server = Process(target=run_auth_server)
    auth_server.start()
    app_server = Process(target=run_app_server)
    app_server.start()
    print("Access http://localhost:8081/app in your browser")

    def sigint_handler(signal, frame):
        print("Terminating servers...")
        auth_server.terminate()
        auth_server.join()
        app_server.terminate()
        app_server.join()

    signal.signal(signal.SIGINT, sigint_handler)

if __name__ == "__main__":
    main()


# coding: utf-8
import enum
from core.network.timeout import Timeout

class EntityTypes(enum.Enum):
    LOADCASE = "loadcase"
    SIMULATION = "simulation"
    TASK = "task"
    STYPE = "submodelType"
    SUBMODEL = "submodel"

class AbstractEntity(object):
    """
    Class containing common behaviour of main CML-Bench entities, such as loadcases, simulations, tasks
    """
    def __init__(self, app_session, identifier):
        self._app_session = app_session
        self._identifier = identifier
        self._http_session = self._app_session.session
        self._sender = self._app_session.sender
        self._handler = self._app_session.handler

        self._entity_type = None
        self._name = None
        self._parent_id = None
        self._tree_path = None
        self._tree_id = None

    def __repr__(self):
        return "Entity type: {} | Entity ID: {}".format(self.entity_type, self.identifier)

    @property
    def entity_type(self):
        return self._entity_type

    @property
    def identifier(self):
        return self._identifier

    @property
    def name(self):
        return self._name

    @property
    def parent_id(self):
        return self._parent_id

    @property
    def tree_path(self):
        return self._tree_path

    @property
    def tree_id(self):
        return self._tree_id

    def _set_entity_type(self, entity_type):
        if isinstance(entity_type, EntityTypes):
            self._entity_type = entity_type

    def _setup_attributes(self):
        if self.entity_type:
            response = self._sender.send_entity_base_info_request(self.identifier, self.entity_type.value)
            Timeout.hold_your_horses()
            self._handler.set_response(response)
            base_info = self._handler.handle_response_to_entity_base_info_request()
            self._name = base_info.get("name")
            self._parent_id = base_info.get("parent_id")
            self._tree_path = base_info.get("tree_path")
            self._tree_id = base_info.get("tree_id")

class Loadcase(AbstractEntity):
    """
    Class for representation of the loadcase entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.LOADCASE)
        self._setup_attributes()

    def get_list_of_simulations(self):
        """
        Method for getting a list of simulations, belonging to the loadcase
        :return: list of simulation objects, or None if some error occurred during reading simulations
        """
        response = self._sender.send_loadcase_simulations_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_ids_list = self._handler.handle_response_to_loadcase_simulations_request()
        if simulation_ids_list:
            simulations = []
            for simulation_id in simulation_ids_list:
                simulations.append(Simulation(self._app_session, simulation_id))
            return simulations
        return None

class Simulation(AbstractEntity):
    """
    Class for representation of the simulation entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.SIMULATION)
        self._setup_attributes()

    def clone(self):
        """
        Method for creating a new simulation, based on the current one
        :return: id of the new simulation, or None if failed to clone simulation
        """
        response = self._sender.send_clone_simulation_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        return self._handler.handle_response_to_clone_simulation_request()

    def get_parent_loadcase(self):
        """
        Method for getting a parent loadcase
        :return: loadcase object
        """
        return Loadcase(self._app_session, self.parent_id)

    def get_list_of_tasks(self):
        """
        Method for getting a list of tasks, belonging to the simulation
        :return: list of task objects, of None if some error occurred during reading tasks
        """
        response = self._sender.send_simulation_tasks_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_tasks_list = self._handler.handle_response_to_simulation_tasks_request()
        if simulation_tasks_list:
            tasks = []
            for task_id in simulation_tasks_list:
                tasks.append(Task(self._app_session, task_id))
            return tasks
        return None

    def get_list_of_submodels(self):
        response = self._sender.send_simulation_submodels_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if simulation_submodels_list:
            submodels = []
            for submodel_id in simulation_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

    def add_new_sumbodels(self, new_submodel_ids):
        # First, get list of current submodels
        response = self._sender.send_simulation_submodels_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if not simulation_submodels_list:
            simulation_submodels_list = []
        # Append new submodel to the list of existing submodels
        simulation_submodels_list.extend(new_submodel_ids)
        # Send request to update simulation submodels (that's how it works in CML-Bench)
        response = self._sender.send_simulation_submodels_update_request(self.identifier, simulation_submodels_list)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        simulation_submodels_list = self._handler.handle_response_to_simulation_submodels_request()
        if simulation_submodels_list:
            submodels = []
            for submodel_id in simulation_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

class Task(AbstractEntity):
    """
    Class for representation of the task entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.TASK)
        self._setup_attributes()

    def get_status(self):
        response = self._sender.send_task_status_request(self.identifier)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        return self._handler.handle_response_to_task_status_response()

class SubmodelType(AbstractEntity):
    """
    Class for representation of the s|type entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.STYPE)
        self._setup_attributes()

    def get_list_of_submodels(self):
        response = self._sender.send_stype_submodels_request(self.tree_path)
        Timeout.hold_your_horses()
        self._handler.set_response(response)
        stype_submodels_list = self._handler.handle_response_to_stype_submodels_requests()
        if stype_submodels_list:
            submodels = []
            for submodel_id in stype_submodels_list:
                submodels.append(Submodel(self._app_session, submodel_id))
            return submodels
        return None

    def upload_new_submodel(self, *files, **params):
        if "stype" in params.keys():
            stype = SubmodelType(self._app_session, params.get("stype"))
        else:
            stype = SubmodelType(self._app_session, self._app_session.cfg.server_storage)

        if "add_to_clipboard" in params.keys():
            add_to_clipboard = "on" if bool(params.get("add_to_clipboard")) else "off"
        else:
            add_to_clipboard = "off"

        submodels = []
        for file in files:
            response = self._sender.send_upload_submodel_request(file, stype.tree_id, add_to_clipboard)
            Timeout.hold_your_horses()
            self._handler.set_response(response)
            sumbodel_id = self._handler.handle_response_to_upload_submodel_request()
            if sumbodel_id:
                submodels.append(Submodel(self._app_session, sumbodel_id))

        return submodels

class Submodel(AbstractEntity):
    """
    Class for representation of the submodel entity
    """
    def __init__(self, app_session, identifier):
        super().__init__(app_session, identifier)
        self._set_entity_type(EntityTypes.SUBMODEL)
        self._setup_attributes()


# To run this, you can install BeautifulSoup
# https://pypi.python.org/pypi/beautifulsoup4

# Or download the file
# http://www.py4e.com/code3/bs4.zip
# and unzip it in the same directory as this file

import urllib.request, urllib.parse, urllib.error
from bs4 import BeautifulSoup
import ssl

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = input('Enter - ')
# url='http://py4e-data.dr-chuck.net/known_by_Fikret.html'
# position = 2
position = input('Enter position:')
position = int(position) - 1
# repeat = 4
repeat = input('Enter Count:')
repeat = int(repeat)

# Retrieve all of the anchor tags
def anchor():
    html = urllib.request.urlopen(url, context=ctx).read()
    soup = BeautifulSoup(html, 'html.parser')
    links=list()
    tags = soup('a')
    for tag in tags:
        # print(tag)
        tag=(tag.get('href', None))
        # print(tag)
        links.append(tag)
    #print(links)
    link=links[position]
    #print('New URL', link)
    return link

while repeat > 0:
    nurl = anchor()
    #print(nurl)
    url=nurl
    print(url)
    repeat=repeat - 1