# Referece https://github.com/icoxfog417/baby-steps-of-rl-ja

import os
import argparse
import io
import re
from collections import namedtuple
from collections import deque
import numpy as np
import tensorflow as tf
from tensorflow.python import keras
from sklearn.preprocessing import StandardScaler
from PIL import Image
import matplotlib.pyplot as plt
import gym
import gym_ple
tf.compat.v1.disable_eager_execution()

Experience = namedtuple("Experience", ["state", "action", "reward", "new_state", "finish_flag"])

class ACAgent():

    def __init__(self, epsilon, actions):
        self.epsilon = epsilon
        self.actions = actions
        self.initialized = False
        self.estimate_probability = False
        self.method_name = "ac"

    @classmethod
    def load(cls, env, model_path, epsilon):
        actions = list(range(env.action_space.n))
        agent = cls(epsilon, actions)
        agent.model = keras.models.load_model(model_path, custom＿objects={
            "SampleAction": SampleAction})
        agent.initialized = True
        return agent

    def initialize(self, experiences, optimizer):
        input_shape = experiences[0].state.shape
        self.create_model(input_shape)
        self.create_updater(optimizer)
        self.initialized = True
        print("Start training")

    def save(self, model_path):
        self.model.save(model_path, overwrite=True, include_optimizer=False)

    def policy(self, state):
        if self.method_name == "ac":
            if not self.initialized:
                return np.random.randint(len(self.actions))
            else:
                action, _, _ = self.model.predict(np.array([state]))
                # actionはlist形式で出るため、0をつけてfloatにする
                # action is a list, so [0] is added to action.
                return action[0]
        else:
            if np.random.random() < self.epsilon or not self.initialized:
                return np.random.randint(len(self.actions))
            else:
                estimates = self.estimate(state)
                if self.estimate_probability:
                    action = np.random.choice(self.actions, size=1, p=estimates)[0]
                    return action
                else:
                    return np.argmax(estimates)

    def estimate(self, state):
        action, action_evaluation, values = self.model.predict(np.array([state]))
        return values[0][0]

    def update(self, states, actions, rewards):
        return self._updater([states, actions, rewards])

    def create_model(self, input_shape):
        normal_initialize = keras.initializers.glorot_normal()
        model = keras.Sequential()
        model.add(keras.layers.Conv2D(
            32, kernel_size=8, strides=4, padding="same",
            input_shape=input_shape,
            kernel_initializer=normal_initialize, activation="relu"
        ))
        model.add(keras.layers.Conv2D(
            64, kernel_size=4, strides=2, padding="same",
            kernel_initializer=normal_initialize, activation="relu"
        ))
        model.add(keras.layers.Conv2D(
            64, kernel_size=3, strides=1, padding="same",
            kernel_initializer=normal_initialize, activation="relu"
        ))
        model.add(keras.layers.Flatten())
        model.add(keras.layers.Dense(256, kernel_initializer=normal_initialize,
                                    activation="relu"))
        # action and critics shares the above model.
        # action model
        actor_layer = keras.layers.Dense(len(self.actions), kernel_initializer=normal_initialize)
        action_evaluation = actor_layer(model.output)
        actions = SampleAction()(action_evaluation)
        # critic model
        critic_layer = keras.layers.Dense(1, kernel_initializer=normal_initialize)
        values = critic_layer(model.output)

        self.model = keras.Model(inputs=model.input, outputs=[actions, action_evaluation, values])

    def create_updater(self, optimizer, value_loss_weight=1.0, entropy_weight=0.1):
        actions = tf.compat.v1.placeholder(shape=(None), dtype="int32")
        values = tf.compat.v1.placeholder(shape=(None), dtype="float32")
        # modelのoutputから各行動確率及び価値を取ってくる
        _, action_evaluation, values_estimation = self.model.output
        cross_logs = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=action_evaluation, labels=actions)
        advantages = values - tf.stop_gradient(values_estimation)  # Q-V
        policy_loss = tf.reduce_mean(cross_logs * advantages)
        value_loss = tf.keras.losses.MeanSquaredError()(values, values_estimation)
        action_entropy = tf.reduce_mean(self.categorical_entropy(action_evaluation))
        loss = policy_loss + value_loss * value_loss_weight
        loss -= entropy_weight * action_entropy

        updates = optimizer.get_updates(loss=loss, params=self.model.trainable_weights)

        self._updater = keras.backend.function(
            inputs=[self.model.input, actions, values],
            outputs=[loss,
                     policy_loss,
                     value_loss,
                     tf.reduce_mean(cross_logs),
                     tf.reduce_mean(advantages),
                     action_entropy],
            updates=updates
        )

    def categorical_entropy(self, logits):
            """
            From OpenAI baseline implementation.
            https://github.com/openai/baselines/blob/master/baselines/common/distributions.py#L192
            """
            a0 = logits - tf.reduce_max(logits, axis=-1, keepdims=True)
            ea0 = tf.exp(a0)
            z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True)
            p0 = ea0 / z0
            return tf.reduce_sum(p0 * (tf.math.log(z0) - a0), axis=-1)

    def play(self, env, episode_count=5, render=True):
        for count in range(episode_count):
            state = env.reset()
            finish_flag = False
            episode_reward = 0
            while not finish_flag:
                if render:
                    env.render()
                action = self.policy(state)
                new_state, reward, finish_flag, info = env.step(action)
                episode_reward += reward
                state = new_state
            else:
                print("Get reward {}".format(episode_reward))

class SampleAction(keras.layers.Layer):
    # keras.layers.Layer クラスを継承
    # override keras.layers.Layer class

    def __init__(self, **kwargs):
        self.output_dim = 1
        super(SampleAction, self).__init__(**kwargs)

    def build(self, input_shape):
        super(SampleAction, self).build(input_shape)

    def call(self, x):
        noise = tf.random.uniform(tf.shape(x))
        return tf.argmax(x - tf.math.log(- tf.math.log(noise)), axis=1)

    def compute_output_shape(self, input_shape):
        return (input_shape[0], self.output_dim)

class ACAgentTest(ACAgent):

    def create_model(self, input_shape):
        normal_initialize = keras.initializers.glorot_normal()
        model = keras.Sequential()
        model.add(keras.layers.Dense(
            10, input_shape=input_shape,
            kernel_initializer=normal_initialize, activation="relu"
        ))
        model.add(keras.layers.Dense(
            10, kernel_initializer=normal_initialize, activation="relu"
        ))
        actor_layer = keras.layers.Dense(
            len(self.actions), kernel_initializer=normal_initialize)
        action_evaluation = actor_layer(model.output)
        actions = SampleAction()(action_evaluation)

        critic_layer = keras.layers.Dense(1, kernel_initializer=normal_initialize)
        values = critic_layer(model.output)

        self.model = keras.Model(
            inputs=model.input, outputs=[actions, action_evaluation, values]
        )

class Observer():

    def __init__(self, env):
        self._env = env

    @property
    def action_space(self):
        return self._env.action_space

    @property
    def observation_space(self):
        return self._env.observatoin_space

    def reset(self):
        return self.transform(self._env.reset())

    def render(self):
        self._env.render(mode="human")

    def step(self, action):
        new_state, reward, finish_flag, info = self._env.step(action)
        return self.transform(new_state), reward, finish_flag, info

    def transform(self, state):
        raise NotImplementedError("Implement transform method.")

class CatcherObserver(Observer):

    def __init__(self, env, width, height, frame_count):
        super().__init__(env)
        self.width = width
        self.height = height
        self.frame_count = frame_count
        self._frames = deque(maxlen=frame_count)

    def transform(self, state):
        # original picture -> new state
        # gray scale (<-- convert("L"))
        grayed_image = Image.fromarray(state).convert("L")
        resized_image = grayed_image.resize((self.width, self.height))
        resized_image = np.array(resized_image).astype("float")
        normalized_image = resized_image / 255.0  # 0 ~ 255 -> 0 ~ 1
        if len(self._frames) == 0:
            # frame数が少ないと学習できないため、frame_count分貯める
            for _ in range(self.frame_count):
                self._frames.append(normalized_image)
        else:
            self._frames.append(normalized_image)
        input_feature = np.array(self._frames)
        # (f, w, h) =(4, 80, 80) set at main() -> (h, w, f) = (80, 80, 4)
        input_feature = np.transpose(input_feature, (1, 2, 0))
        return input_feature

class Trainer():

    def __init__(self, buffer_size=1024, batch_size=33,
                 gamma=0.9, report_interval=10, log_dir=""):
        self.buffer_size = buffer_size
        self.batch_size = batch_size
        self.gamma = gamma
        self.report_interval = report_interval
        self.logger = Logger(log_dir, self.trainer_name)
        self.experiences = deque(maxlen=buffer_size)
        self.training = False
        self.training_count = 0
        self.reward_log = []

    @property
    def trainer_name(self):
        class_name = self.__class__.__name__
        name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", class_name)
        name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower()
        name = name.replace("_trainer", "")
        return name

    def train_loop(self, env, agent, episode=200, initial_count=-1,
                    render=False, observe_interval=0):
        self.experiences = deque(maxlen=self.buffer_size)
        self.training = False
        self.training_count = 0
        self.reward_log = []
        frames = []

        for i in range(episode):
            state = env.reset()
            finish_flag = False
            step_count = 0
            self.episode_begin(i, agent)
            while not finish_flag:
                if render:
                    env.render()
                if self.training and observe_interval > 0 and  \
                    (self.training_count == 1 or
                        self.training_count % observe_interval == 0):
                    frames.append(state)
                agent_action = agent.policy(state)
                new_state, reward, finish_flag, info = env.step(agent_action)
                experience = Experience(state, agent_action, reward, new_state, finish_flag)
                self.experiences.append(experience)
                if not self.training and \
                    len(self.experiences) == self.buffer_size:
                    # Strat training
                    self.begin_train(i, agent)
                    self.training = True
                self.step(i, step_count, agent, experience)
                state = new_state
                step_count += 1
            else:
                self.episode_end(i, step_count, agent)

                if not self.training and \
                    initial_count > 0 and i >= initial_count:
                    self.begin_train(i, agent)
                    self.training = True

                if self.training:
                    if len(frames) > 0:
                        self.logger.write_image(self.training_count, frames)
                        frames = []
                    self.training_count += 1

    def episode_begin(self, episode, agent):
        pass

    def begin_train(self, episode, agent):
        pass

    def step(self, episode, step_count, agent, experience):
        pass

    def episode_end(self, episode, step_count, agent):
        pass

    def is_event(self, count, interval):
        return True if count != 0 and count % interval == 0 else False

    def get_recent(self, count):
        recent = range(len(self.experiences) - count, len(self.experiences))
        return [self.experiences[i] for i in recent]

class ACTrainer(Trainer):

    def __init__(self, buffer_size=256, batch_size=32, gamma=0.99,
                 learning_rate=1e-3, report_interval=10, log_dir="", file_name=""):
        super().__init__(buffer_size, batch_size, gamma, report_interval, log_dir)
        self.file_name = file_name if file_name else "a2c_agent.h5"
        self.learning_rate = learning_rate
        self.losses = {}
        self.rewards = []
        self._max_reward = -10

    def train(self, env, episode_count=900, initial_count=10,
              test_mode=False, render=False, observe_interval=100, epsilon=0.0):
        actions = list(range(env.action_space.n))
        if not test_mode:
            agent = ACAgent(epsilon, actions)
        else:
            agent = ACAgentTest(epsilon, actions)
            observe_interval = 0
        self.training_episode = episode_count
        self.train_loop(env, agent, episode_count, initial_count, render,observe_interval)

        return agent

    def episode_begin(self, episode, agent):
        self.rewards = []

    def step(self, episode, step_count, agent, experience):
        self.rewards.append(experience.reward)
        if not agent.initialized:
            if len(self.experiences) < self.buffer_size:
                return False

            optimizer = keras.optimizers.Adam(lr=self.learning_rate,clipnorm=5.0)
            agent.initialize(self.experiences, optimizer)
            self.logger.set_model(agent.model)
            self.training = True
            self.experiences.clear()
        else:
            if len(self.experiences) < self.batch_size:
                return False

            batch = self.make_batch(agent)
            loss, lp, lv, p_ng, p_ad, p_en = agent.update(*batch)
            self.losses["loss/total"] = loss
            self.losses["loss/policy"] = lp
            self.losses["loss/value"] = lv
            self.losses["policy/neg_logs"] = p_ng
            self.losses["policy/advantage"] = p_ad
            self.losses["policy/entropy"] = p_en
            self.experiences.clear()

    def make_batch(self, agent):
        states = []
        actions = []
        values = []
        experiences = list(self.experiences)
        states = np.array([experience.state for experience in experiences])
        actions=np.array([experience.action for experience in experiences])
        last_experience = experiences[-1]
        if last_experience.finish_flag:
            future_reward = last_experience.reward
        else:
            future_reward = agent.estimate(last_experience.new_state)
        for experience in reversed(experiences):
            value = experience.reward
            if not experience.finish_flag:
                value += self.gamma * future_reward
            values.append(value)
            future_reward = value
        values = np.array(list(reversed(values)))
        scaler = StandardScaler()
        values = scaler.fit_transform(values.reshape((-1, 1))).flatten()
        return states, actions, values

    def episode_end(self, episode, step_count, agent):
        reward = sum(self.rewards)
        self.reward_log.append(reward)

        if agent.initialized:
            self.logger.write(self.training_count, "reward", reward)
            self.logger.write(self.training_count, "reward_max",
                              max(self.rewards))

            for k in self.losses:
                self.logger.write(self.training_count, k, self.losses[k])

            if reward > self._max_reward:
                agent.save(self.logger.path_of(self.file_name))
                self._max_reward = reward

        if self.is_event(episode, self.report_interval):
            recent_rewards = self.reward_log[-self.report_interval:]
            self.logger.describe("reward", recent_rewards, episode=episode)

class Logger():

    def __init__(self, log_dir="", dir_name=""):
        self.log_dir = log_dir
        if not log_dir:
            self.log_dir = os.path.join(os.path.dirname(__file__), "logs")
        if not os.path.exists(self.log_dir):
            os.mkdir(self.log_dir)

        if dir_name:
            self.log_dir = os.path.join(self.log_dir, dir_name)
            if not os.path.exists(self.log_dir):
                os.mkdir(self.log_dir)

        self._callback = tf.compat.v1.keras.callbacks.TensorBoard(
            self.log_dir)

    @property
    def writer(self):
        return self._callback.writer

    def set_model(self, model):
        self._callback.set_model(model)

    def path_of(self, file_name):
        return os.path.join(self.log_dir, file_name)

    def describe(self, name, values, episode=-1, step=-1):
        mean = np.round(np.mean(values), 3)
        std = np.round(np.std(values), 3)
        desc = "{} is {} (+/-{})".format(name, mean, std)
        if episode > 0:
            print("At episode {}, {}".format(episode, desc))
        elif step > 0:
            print("At step {}, {}".format(step, desc))

    def plot(self, name, values, interval=10):
        indices = list(range(0, len(values), interval))
        means = []
        stds = []
        for i in indices:
            _values = values[i:(i + interval)]
            means.append(np.mean(_values))
            stds.append(np.std(_values))
        means = np.array(means)
        stds = np.array(stds)
        plt.figure()
        plt.title("{} History".format(name))
        plt.grid()
        plt.fill_between(indices, means - stds, means + stds,
                         alpha=0.1, color="g")
        plt.plot(indices, means, "o-", color="g",
                 label="{} per {} episode".format(name.lower(), interval))
        plt.legend(loc="best")
        plt.show()

    def write(self, index, name, value):
        summary = tf.compat.v1.Summary()
        summary_value = summary.value.add()
        summary_value.tag = name
        summary_value.simple_value = value
        self.writer.add_summary(summary, index)
        self.writer.flush()

    def write_image(self, index, frames):
        # Deal with a 'frames' as a list of sequential gray scaled image.
        last_frames = [f[:, :, -1] for f in frames]
        if np.min(last_frames[-1]) < 0:
            scale = 127 / np.abs(last_frames[-1]).max()
            offset = 128
        else:
            scale = 255 / np.max(last_frames[-1])
            offset = 0
        channel = 1  # gray scale
        tag = "frames_at_training_{}".format(index)
        values = []

        for f in last_frames:
            height, width = f.shape
            array = np.asarray(f * scale + offset, dtype=np.uint8)
            image = Image.fromarray(array)
            output = io.BytesIO()
            image.save(output, format="PNG")
            image_string = output.getvalue()
            output.close()
            image = tf.compat.v1.Summary.Image(
                height=height, width=width, colorspace=channel,
                encoded_image_string=image_string)
            value = tf.compat.v1.Summary.Value(tag=tag, image=image)
            values.append(value)

        summary = tf.compat.v1.Summary(value=values)
        self.writer.add_summary(summary, index)
        self.writer.flush()

def main(play, is_test):
    file_name = "a2c_agent.h5" if not is_test else "a2c_agent_test.h5"
    trainer = ACTrainer(file_name=file_name)
    path = trainer.logger.path_of(trainer.file_name)
    agent_class = ACAgent
    epsilon = 0.0

    if is_test:
        print("Test mode")
        observer = gym.make("CartPole-v0")
        agent_class = ACAgentTest
    else:
        env_ = gym.make("Catcher-v0")
        observer = CatcherObserver(env_, 80, 80, 4)
        trainer.learning_rate = 5e-5

    if play:
        agent = agent_class.load(observer, path, epsilon)
        agent.play(observer, episode_count=10, render=True)
    else:
        trainer.train(observer, test_mode=is_test, epsilon=epsilon)
    observer.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="A2C")
    parser.add_argument("--play", action="store_true")
    parser.add_argument("--test", action="store_true")
    args = parser.parse_args()
    main(args.play, args.test)


"""
Test file for vanilla DQN
"""
#%% Import
from game_motor.experiment import Experiment, process_game_statistics
from game_motor.actions_builder import Action
from game_motor.reward import Reward
from DQN import DQN_agent
import matplotlib.pyplot as plt
import pickle

#%% Logger
import logging
from logging import FileHandler

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
file_handler = FileHandler('activity_dqn_basic_1.log')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

#%% Define the environment
list_action =[
		'TURN_LEFT',
		'TURN_RIGHT',
		'MOVE_FORWARD']

screen_resolution = 'RES_160X120'
screen_format='GRAY8'
living_reward = 1

action_builder = Action(list_action) # create actions

reward = {'medikit' : 100.}

reward_builder = Reward(dic_reward=reward, default=False) # default rewards
print(reward_builder.default_reward)
custom_reward=True

game_features = ['health']
game_variables = ['HEALTH']

scenario = 'D1_basic'

experiment = Experiment(scenario, action_builder,reward_builder, logger,
               custom_reward=custom_reward,living_reward=living_reward,
               game_features = game_features, visible=False,
               screen_format = screen_format)

#%% Define the agent
screen_shape = (84,84)
depth = 4
image_params = {'screen_input_size' : screen_shape + (depth,)}
n_actions = action_builder.n_actions
#decrease_eps = lambda step : 0.02 + 145000. / (float(step) + 150000.)

#def exploration_rate(epoch, nb_episode):
#        """# Define exploration rate change over time"""
#        start_eps = 1.0
#        end_eps = 0.1
#        const_eps_epochs = 0.1 * nb_episode # 10% of learning time
#        eps_decay_epochs = 0.6 * nb_episode  # 60% of learning time
#
#        if epoch < const_eps_epochs:
#            return start_eps
#        elif epoch < eps_decay_epochs:
#            # Linear decay
#            return start_eps + (epoch - const_eps_epochs)/(const_eps_epochs -
#                                           eps_decay_epochs) * (start_eps - end_eps)
#        else:
#            return end_eps

map_id = 1
nb_episodes = 20000
nb_episodes_test = 5
#decrease_eps = lambda eps : exploration_rate(eps, nb_episodes)
decrease_eps = lambda step : 0.02 + 145000. / (float(step) + 150000.)

agent = DQN_agent(image_params, n_actions, logger, decrease_eps=decrease_eps)

#%% Run the training, then testing
agent.train(map_id, experiment, nb_episodes)


import feature_utils
import numpy as np
import pandas as pd
import math
import os
import subprocess
import librosa
import ast
import essentia
import essentia.standard as es
from scipy import signal, interpolate
from scipy.io import wavfile
from pyvad import vad
from sklearn import preprocessing
from gensim.models import Word2Vec
import random
from string import digits
from pocketsphinx import AudioFile

dtype_params = {'dx_pitch': float,
                'dx_intensity': float}
dtype_pitch_tier = {'Time': float,
                    'F0': float}
dtype_intensity_tier = {'Time': float,
                        'Intensity': float}
dtype_markvowels = {'tmin': float,
                    'tmax': float}
dtype_voice_report_old = {'start_time': float,
                          'end_time': float,
                          'harmonicity': str,
                          'jitter': str,
                          'shimmer': str}
dtype_voice_report = {'start_time': float,
                      'end_time': float,
                      'harmonicity': float,
                      'jitter': float,
                      'shimmer': float}
dtype_silences = {'tmin': float,
                  'text': str,
                  'tmax': float}
syllables_dtype = {'tmin': float,
                   'tier': str}
transcription_and_diarization_dtype = {'word': str,
                                       'start_time': float,
                                       'end_time': float,
                                       'speaker_tag': str}

def get_noise_suppressed_audio(audio_fn, config_parser):
    if config_parser.getboolean('NOISE_SUPPRESSION_TOOL', 'is_noiseless'):
        return audio_fn
    audio_ext = config_parser.get('GLOBAL', 'audio_ext')
    audio_ext_len = len(audio_ext)
    base_audio_fn = os.path.basename(audio_fn)[:-audio_ext_len]
    dest_folder = config_parser.get('GLOBAL', 'dest_folder')
    base_name = dest_folder + base_audio_fn + '/' + base_audio_fn
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    noise_suppression_tool = config_parser.get('GLOBAL', 'noise_suppression_tool')
    ffmpeg = config_parser.get('GLOBAL', 'ffmpeg')
    nst_raw_audio_suffix = config_parser.get('NOISE_SUPPRESSION_TOOL', 'raw_audio_suffix')
    nst_raw_audio_ext = config_parser.get('NOISE_SUPPRESSION_TOOL', 'raw_audio_ext')
    nst_raw_audio_fn = base_name + nst_raw_audio_suffix + nst_raw_audio_ext
    dns_audio_suffix = config_parser.get('NOISE_SUPPRESSION_TOOL', 'denoised_audio_suffix')
    dns_audio_fn = base_name + dns_audio_suffix + audio_ext
    nst_sample_rate = config_parser.getint('NOISE_SUPPRESSION_TOOL', 'nst_sample_rate')
    ffmpeg_raw_conversion_format_option = config_parser.get('FFMPEG', 'raw_conversion_format_option')
    ffmpeg_channels_number = config_parser.get('FFMPEG', 'channels_number')
    if audio_sample_rate == nst_sample_rate:
        subprocess.call([noise_suppression_tool, audio_fn, nst_raw_audio_fn])
        subprocess.call(ffmpeg,
                        '-f', ffmpeg_raw_conversion_format_option,
                        '-ar', str(audio_sample_rate),
                        '-ac', ffmpeg_channels_number,
                        '-i', nst_raw_audio_fn,
                        dns_audio_fn)
    else:
        ups_audio_suffix = config_parser.get('FFMPEG', 'upsampled_audio_suffix')
        ups_audio_fn = base_name + ups_audio_suffix + audio_ext
        ups_den_audio_suffix = config_parser.get('NOISE_SUPPRESSION_TOOL', 'upsampled_denoised_audio_suffix')
        ups_den_audio_fn = base_name + ups_den_audio_suffix + audio_ext
        subprocess.call([ffmpeg,
                         '-i', audio_fn,
                         '-ar', str(nst_sample_rate),
                         '-ac', ffmpeg_channels_number,
                         ups_audio_fn])
        subprocess.call([noise_suppression_tool, ups_audio_fn, nst_raw_audio_fn])
        subprocess.call([ffmpeg,
                         '-f', ffmpeg_raw_conversion_format_option,
                         '-ar', str(nst_sample_rate),
                         '-ac', ffmpeg_channels_number,
                         '-i', nst_raw_audio_fn,
                         ups_den_audio_fn])
        subprocess.call([ffmpeg,
                         '-i', ups_den_audio_fn,
                         '-ar', str(audio_sample_rate),
                         '-ac', ffmpeg_channels_number,
                         dns_audio_fn])
    return dns_audio_fn

def get_praat_features(audio_fn, config_parser):
    audio_ext = config_parser.get('GLOBAL', 'audio_ext')
    audio_ext_len = len(audio_ext)
    base_audio_fn = os.path.basename(audio_fn)[:-audio_ext_len]
    dest_folder = config_parser.get('GLOBAL', 'dest_folder')
    base_name = dest_folder + base_audio_fn + '/' + base_audio_fn
    praat = config_parser.get('GLOBAL', 'praat')
    praat_tab_ext = config_parser.get('PRAAT', 'tab_ext')
    script = config_parser.get('PRAAT', 'features_script')
    params_table_suffix = config_parser.get('PRAAT', 'params_suffix')
    params_fn = base_name + params_table_suffix + praat_tab_ext
    pitch_tier_suffix = config_parser.get('PRAAT', 'pitch_tier_suffix')
    pitch_tier_fn = base_name + pitch_tier_suffix + praat_tab_ext
    intensity_tier_suffix = config_parser.get('PRAAT', 'intensity_tier_suffix')
    intensity_tier_fn = base_name + intensity_tier_suffix + praat_tab_ext
    markvowels_table_suffix = config_parser.get('PRAAT', 'markvowels_table_suffix')
    markvowels_table_fn = base_name + markvowels_table_suffix + praat_tab_ext
    voice_report_suffix = config_parser.get('PRAAT', 'voice_report_suffix')
    voice_report_fn = base_name + voice_report_suffix + praat_tab_ext
    silences_table_suffix = config_parser.get('PRAAT', 'silences_table_suffix')
    silences_table_fn = base_name + silences_table_suffix + praat_tab_ext
    syllables_table_suffix = config_parser.get('PRAAT', 'syllables_table_suffix')
    syllables_table_fn = base_name + syllables_table_suffix + praat_tab_ext
    pitch_min = config_parser.get('PRAAT', 'pitch_min')
    pitch_max = config_parser.get('PRAAT', 'pitch_max')
    max_period_factor = config_parser.get('PRAAT', 'max_period_factor')
    max_amplitude_factor = config_parser.get('PRAAT', 'max_amplitude_factor')
    silence_threshold = config_parser.get('PRAAT', 'silence_threshold')
    voicing_threshold = config_parser.get('PRAAT', 'voicing_threshold')
    silence_threshold_db = config_parser.get('PRAAT', 'silence_threshold_db')
    minimum_silent_interval_duration = config_parser.get('PRAAT', 'minimum_silent_interval_duration')
    minimum_sounding_interval_duration = config_parser.get('PRAAT', 'minimum_sounding_interval_duration')
    silence_threshold_syl = config_parser.get('PRAAT', 'silence_threshold_syl')
    minimum_dip_between_peaks_syl = config_parser.get('PRAAT', 'minimum_dip_between_peaks_syl')
    minimum_pause_duration_syl = config_parser.get('PRAAT', 'minimum_pause_duration_syl')
    time_decimals = config_parser.get('PRAAT', 'time_decimals')
    subprocess.call([praat,
                     '--run',
                     script,
                     audio_fn,
                     params_fn,
                     pitch_tier_fn,
                     intensity_tier_fn,
                     markvowels_table_fn,
                     voice_report_fn,
                     silences_table_fn,
                     syllables_table_fn,
                     pitch_min,
                     pitch_max,
                     max_period_factor,
                     max_amplitude_factor,
                     silence_threshold,
                     voicing_threshold,
                     silence_threshold_db,
                     minimum_silent_interval_duration,
                     minimum_sounding_interval_duration,
                     silence_threshold_syl,
                     minimum_dip_between_peaks_syl,
                     minimum_pause_duration_syl,
                     time_decimals])
    # PostProcess voice report before returning
    voice_report_df = pd.read_csv(voice_report_fn, dtype=dtype_voice_report_old)
    voice_report_df = voice_report_df[voice_report_df.harmonicity != '--undefined--']
    voice_report_df = voice_report_df[voice_report_df.jitter != '--undefined--']
    voice_report_df = voice_report_df[voice_report_df.shimmer != '--undefined--']
    voice_report_df.to_csv(voice_report_fn, index=False)
    return (params_fn, pitch_tier_fn, intensity_tier_fn, markvowels_table_fn,
            voice_report_fn, silences_table_fn, syllables_table_fn)

def get_text_transcription(audio_fn, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    audio_ext = config_parser.get('GLOBAL', 'audio_ext')
    audio_ext_len = len(audio_ext)
    base_audio_fn = os.path.basename(audio_fn)[:-audio_ext_len]
    dest_folder = config_parser.get('GLOBAL', 'dest_folder')
    base_name = dest_folder + base_audio_fn + '/' + base_audio_fn
    text_transcription_suffix = config_parser.get('ASR', 'text_transcription_suffix')
    tab_ext = config_parser.get('ASR', 'tab_ext')
    text_transcription_fn = base_name + text_transcription_suffix + tab_ext
    language = config_parser.get('ASR', 'language')
    speakers_n = config_parser.get('ASR', 'speakers_n')
    words_info_df_columns = ast.literal_eval(config_parser.get('ASR', 'words_info_df_columns'))
    ########################
    frate = 100
    forbidden = ('<s>', '</s>', '<sil>')
    speech_tag = '[SPEECH]'
    speaker_tags = ['speaker_'+str(i) for i in range(speakers_n)]
    audio = AudioFile(audio_file=audio_fn, frate=frate)
    words_info = []
    for phrase in audio:
        for segment in phrase.segments(detailed=True):
            (word, p, start_time, end_time) = segment
            start_time = start_time / frate
            end_time = end_time / frate
            speaker_tag = random.choice(speaker_tags)
            if word == speech_tag:
                words_info.append([word, start_time, end_time, speaker_tag])
            elif word not in forbidden:
                remove_digits = str.maketrans('', '', digits)
                word = word.translate(remove_digits)
                words_info.append([word, start_time, end_time, speaker_tag])
    words_info_df = pd.DataFrame(data=words_info, columns=words_info_df_columns)
    words_info_df.to_csv(text_transcription_fn, index=False)
    return text_transcription_fn

def get_padded_audio(audio_fn, config_parser):
    sr, audio = wavfile.read(audio_fn)
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    pad_width_samples = feature_hop_length_samples - ((len_audio_samples - feature_window_length_samples) % feature_hop_length_samples)
    if pad_width_samples == 0:
        return essentia.array(audio)
    else:
        return essentia.array(np.pad(audio, (0, pad_width_samples), 'constant', constant_values=0))

def get_voiced_filters(audio, silences_table_fn, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    silences_df = pd.read_csv(silences_table_fn, dtype=dtype_silences)
    # Extract silences from Praat results
    silent = np.zeros(audio.shape[0])
    only_silent_df = silences_df[silences_df['text'] == 'silent']
    for t_st, t_end in only_silent_df[['tmin', 'tmax']].values:
        t_st_samples = math.floor(t_st * audio_sample_rate)
        t_end_samples = math.floor(t_end * audio_sample_rate)
        silent[t_st_samples:t_end_samples] = 1
    # Extract voiced part as opposite of silences
    voiced = np.abs(silent - 1)
    # Perform voice activity detection
    fs_vad = config_parser.getint('VAD', 'fs_vad')
    hop_length_vad = config_parser.getint('VAD', 'hop_length_vad')
    mode_vad = config_parser.getint('VAD', 'mode_vad')
    detected_voiced = vad(audio * voiced, audio_sample_rate, fs_vad=fs_vad, hoplength=hop_length_vad, vad_mode=mode_vad)
    # Filter VAD results to delete last noisy intervals
    tukey_win_size = config_parser.getfloat('VAD', 'tukey_win_size')
    tukey_win_convolution_mode = config_parser.get('VAD', 'tukey_win_convolution_mode')
    voiced_joiner_threshold = config_parser.getfloat('VAD', 'voiced_joiner_threshold')
    tukey_win_size_samples = math.floor(tukey_win_size * audio_sample_rate)
    tukey_win = signal.tukey(tukey_win_size_samples)
    convolved_voiced = signal.convolve(detected_voiced, tukey_win, mode=tukey_win_convolution_mode) / sum(tukey_win)
    convolved_voiced[np.where(convolved_voiced[:] < voiced_joiner_threshold)] = 0
    convolved_voiced[np.where(convolved_voiced[:] >= voiced_joiner_threshold)] = 1
    if config_parser.getboolean('VAD', 'are_voiced_regions_split'):
        voiced = convolved_voiced * detected_voiced
    else:
        voiced = convolved_voiced
    silent = np.abs(voiced - 1)
    return (voiced, silent)

def get_resampled_filter(filter_original_sr, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_filter_original_sr_samples = filter_original_sr.shape[0]
    len_filter_samples = math.floor((len_filter_original_sr_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    time_original_sr = np.linspace(0, len_filter_original_sr_samples / audio_sample_rate, len_filter_original_sr_samples)
    time = np.linspace(0, len_filter_original_sr_samples / audio_sample_rate, len_filter_samples)
    filter_interpolator = interpolate.interp1d(time_original_sr, filter_original_sr, kind='nearest', fill_value='nearest')
    return filter_interpolator(time)

def get_speaker_filters(transcription_and_diarization_fn, len_audio_samples, voiced_filter, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    transcription_and_diarization_df = pd.read_csv(transcription_and_diarization_fn, dtype=transcription_and_diarization_dtype)
    speaker_filters = []
    speakers = transcription_and_diarization_df.speaker_tag.unique()
    for speaker in speakers:
        df = transcription_and_diarization_df[transcription_and_diarization_df['speaker_tag'] == speaker]
        speaker_filter = np.zeros(len_audio_samples)
        for t_st, t_end in df[['start_time', 'end_time']].values:
            t_st_samples = math.floor(t_st * audio_sample_rate)
            t_end_samples = math.floor(t_end * audio_sample_rate)
            speaker_filter[t_st_samples:t_end_samples] = 1
        speaker_filters.append(speaker_filter)
    remaining = voiced_filter.copy()
    for filter in speaker_filters:
        remaining = remaining * np.abs(filter - 1)
    speaker_filters.append(remaining)
    return np.array(speaker_filters)

def get_resampled_filters(filters_original_sr, config_parser):
    filters = []
    for filter_original_sr in filters_original_sr:
        filters.append(get_resampled_filter(filter_original_sr, config_parser))
    return np.array(filters)

def get_pitch(pitch_tier_fn, len_audio_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    config_section = config_parser['PITCH']
    pitch_df = pd.read_csv(pitch_tier_fn, dtype=dtype_pitch_tier)
    time_pitch_values = pitch_df[['Time', 'F0']].values
    (pitch, delta_pitch, delta_delta_pitch) = feature_utils.get_praat_feature(time_pitch_values, len_audio_samples, audio_sample_rate, feature_sample_rate, feature_window_length_samples, feature_hop_length_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_section)
    return (pitch, delta_pitch, delta_delta_pitch)

def get_intensity(intensity_tier_fn, len_audio_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    config_section = config_parser['INTENSITY']
    intensity_df = pd.read_csv(intensity_tier_fn, dtype=dtype_intensity_tier)
    time_intensity_values = intensity_df[['Time', 'Intensity']].values
    (intensity, delta_intensity, delta_delta_intensity) = feature_utils.get_praat_feature(time_intensity_values, len_audio_samples, audio_sample_rate, feature_sample_rate, feature_window_length_samples, feature_hop_length_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_section)
    return (intensity, delta_intensity, delta_delta_intensity)

def get_voice_report(voice_report_fn, len_audio_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    voice_report_df = pd.read_csv(voice_report_fn, dtype=dtype_voice_report)
    # Reorganize voice report in more data frames
    harmonicity_df = voice_report_df[['start_time', 'end_time', 'harmonicity']]
    jitter_df = voice_report_df[['start_time', 'end_time', 'jitter']]
    shimmer_df = voice_report_df[['start_time', 'end_time', 'shimmer']]
    time_harmonicity_values = feature_utils.gen_voice_report_time_feature_values(harmonicity_df, feature_hop_length, 'harmonicity')
    harmonicty_config_section = config_parser['HARMONICITY']
    time_jitter_values = feature_utils.gen_voice_report_time_feature_values(jitter_df, feature_hop_length, 'jitter')
    jitter_config_section = config_parser['JITTER']
    time_shimmer_values = feature_utils.gen_voice_report_time_feature_values(shimmer_df, feature_hop_length, 'shimmer')
    shimmer_config_section = config_parser['SHIMMER']
    (harmonicity, delta_harmonicity, delta_delta_harmonicity) = feature_utils.get_praat_feature(time_harmonicity_values, len_audio_samples, audio_sample_rate, feature_sample_rate, feature_window_length_samples, feature_hop_length_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, harmonicty_config_section)
    (jitter, delta_jitter, delta_delta_jitter) = feature_utils.get_praat_feature(time_jitter_values, len_audio_samples, audio_sample_rate, feature_sample_rate, feature_window_length_samples, feature_hop_length_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, jitter_config_section)
    (shimmer, delta_shimmer, delta_delta_shimmer) = feature_utils.get_praat_feature(time_shimmer_values, len_audio_samples, audio_sample_rate, feature_sample_rate, feature_window_length_samples, feature_hop_length_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, shimmer_config_section)
    return (harmonicity, delta_harmonicity, delta_delta_harmonicity, jitter, delta_jitter, delta_delta_jitter, shimmer, delta_shimmer, delta_delta_shimmer)

def get_chomagram(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['CHROMAGRAM']
    chromagram = librosa.feature.chroma_stft(y=audio, sr=audio_sample_rate, n_fft=feature_window_length_samples, hop_length=feature_hop_length_samples)
    chromagram = chromagram[:, 1:-1]
    for i in range(chromagram.shape[0]):
        time_chromagram_values = np.array(list(zip(time, chromagram[i])))
        chromagram[i] = feature_utils.get_feature(time_chromagram_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return chromagram

def get_mfccs_and_mel_bands(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    mfcc_config_section = config_parser['MFCC']
    mel_bands_config_section = config_parser['MEL_BANDS']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    mfcc = es.MFCC()
    log_norm = es.UnaryOperator(type='log')
    mfccs = []
    mel_bands = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        mfcc_bands, mfcc_coeffs = mfcc(spec)
        mfccs.append(mfcc_coeffs)
        if config_parser.getboolean('MEL_BANDS', 'take_log'):
            mel_bands.append(log_norm(mfcc_bands))
        else:
            mel_bands.append(mfcc_bands)
    mfccs = np.array(mfccs).T
    mel_bands = np.array(mel_bands).T
    for i in range(mfccs.shape[0]):
        time_mfcc_values = np.array(list(zip(time, mfccs[i])))
        mfccs[i] = feature_utils.get_feature(time_mfcc_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, mfcc_config_section)
    for i in range(mel_bands.shape[0]):
        time_mel_band_values = np.array(list(zip(time, mel_bands[i])))
        mel_bands[i] = feature_utils.get_feature(time_mel_band_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, mel_bands_config_section)
    return (mfccs, mel_bands)

def get_short_term_energy(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SHORT_TERM_ENERGY']
    energy = es.Energy()
    short_term_energy = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        short_term_energy.append(energy(frame))
    short_term_energy = np.array(short_term_energy)
    time_short_term_energy_values = np.array(list(zip(time, short_term_energy)))
    short_term_energy = feature_utils.get_feature(time_short_term_energy_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return short_term_energy

def get_short_term_entropy(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SHORT_TERM_ENTROPY']
    entropy = es.Entropy()
    short_term_entropy = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        short_term_entropy.append(entropy(np.power(frame, 2)))
    short_term_entropy = np.array(short_term_entropy)
    time_short_term_entropy_values = np.array(list(zip(time, short_term_entropy)))
    short_term_entropy = feature_utils.get_feature(time_short_term_entropy_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return short_term_entropy

def get_spectral_centroid(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SPECTRAL_CENTROID']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    centroid_t = es.SpectralCentroidTime(sampleRate=audio_sample_rate)
    spectral_centroid = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        spectral_centroid.append(centroid_t(spec))
    spectral_centroid = np.array(spectral_centroid)
    time_spectral_centroid_values = np.array(list(zip(time, spectral_centroid)))
    spectral_centroid = feature_utils.get_feature(time_spectral_centroid_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return spectral_centroid

def get_spectral_spread(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SPECTRAL_SPREAD']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    spectral_spread = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        spectral_spread.append(np.std(spec))
    spectral_spread = np.array(spectral_spread)
    time_spectral_spread_values = np.array(list(zip(time, spectral_spread)))
    spectral_spread = feature_utils.get_feature(time_spectral_spread_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return spectral_spread

def get_spectral_entropy(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SPECTRAL_ENTROPY']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    entropy = es.Entropy()
    spectral_entropy = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        spectrum_length_samples = spec.shape[0]
        power_spectrum = es.PowerSpectrum(size=spectrum_length_samples)
        power_spectral_density = power_spectrum(spec)  # Spectrum must have even length
        power_spectral_density = preprocessing.maxabs_scale(power_spectral_density)
        spectral_entropy.append(entropy(power_spectral_density))
    spectral_entropy = np.array(spectral_entropy)
    time_spectral_entropy_values = np.array(list(zip(time, spectral_entropy)))
    spectral_entropy = feature_utils.get_feature(time_spectral_entropy_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return spectral_entropy

def get_spectral_flux(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SPECTRAL_FLUX']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    flux = es.Flux()
    spectral_flux = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        spectral_flux.append(flux(spec))
    spectral_flux = np.array(spectral_flux)
    time_spectral_flux_values = np.array(list(zip(time, spectral_flux)))
    spectral_flux = feature_utils.get_feature(time_spectral_flux_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return spectral_flux

def get_spectral_roll_off(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['SPECTRAL_ROLL_OFF']
    spectrum = es.Spectrum(size=feature_window_length_samples)
    roll_off = es.RollOff(sampleRate=audio_sample_rate)
    spectral_roll_off = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        spec = spectrum(frame)
        spectral_roll_off.append(roll_off(spec))
    spectral_roll_off = np.array(spectral_roll_off)
    time_spectral_roll_off_values = np.array(list(zip(time, spectral_roll_off)))
    spectral_roll_off = feature_utils.get_feature(time_spectral_roll_off_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return spectral_roll_off

def get_zero_crossing_rate(audio, voiced_filter, speaker_filters, config_parser):
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    config_section = config_parser['ZERO_CROSSING_RATE']
    zero_crossing = es.ZeroCrossingRate()
    zero_crossing_rate = []
    for frame in es.FrameGenerator(audio, frameSize=feature_window_length_samples, hopSize=feature_hop_length_samples, lastFrameToEndOfFile=False, startFromZero=True):
        zero_crossing_rate.append(zero_crossing(frame))
    zero_crossing_rate = np.array(zero_crossing_rate)
    time_zero_crossing_rate_values = np.array(list(zip(time, zero_crossing_rate)))
    zero_crossing_rate = feature_utils.get_feature(time_zero_crossing_rate_values, len_feature_samples, feature_sample_rate, voiced_filter, speaker_filters, config_section)
    return zero_crossing_rate

def get_onset_and_onset_rate(audio, voiced_filter, config_parser):
    # Those features are handled differently
    len_audio_samples = audio.shape[0]
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    onset_frames = librosa.onset.onset_detect(y=audio, sr=audio_sample_rate, hop_length=feature_hop_length_samples)
    onset = np.zeros(len_feature_samples)
    onset[onset_frames] = 1
    onset = onset * voiced_filter
    # NOTE manage correctly onset_rate
    onset_rate_config_section = config_parser['ONSET_RATE']
    onset_rate_window_length = config_parser.getfloat('ONSET_RATE', 'onset_rate_window')
    onset_rate_window_length_feature_samples = math.floor(onset_rate_window_length * feature_sample_rate)
    onset_rate = np.zeros(len_feature_samples)
    for i in range(len_feature_samples - onset_rate_window_length_feature_samples + 1):
        onset_rate[i] = np.sum(onset[i:i + onset_rate_window_length_feature_samples]) / onset_rate_window_length
    onset_rate[len_feature_samples - onset_rate_window_length_feature_samples + 1:] = onset_rate[len_feature_samples - onset_rate_window_length_feature_samples]
    onset_rate = onset_rate * voiced_filter
    return (onset, onset_rate)

def get_speech_rate(syllables_table_fn, len_audio_samples, voiced_filter_original_sr, voiced_filter, speaker_filters, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    feature_sample_rate = math.floor(1 / feature_hop_length)
    speech_rate_window_length = config_parser.getfloat('SPEECH_RATE', 'speech_rate_window')
    speech_rate_window_length_samples = math.floor(speech_rate_window_length * audio_sample_rate)
    speech_rate_window_length_feature_samples = math.floor(speech_rate_window_length * feature_sample_rate)
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_audio_samples)
    syllables_df = pd.read_csv(syllables_table_fn, dtype=syllables_dtype)
    syllables_time_stamps = np.array([math.floor(x * audio_sample_rate) for x in syllables_df['tmin'].values])
    syllables_time_stamps = syllables_time_stamps[feature_utils.get_valid_time_stamps(voiced_filter_original_sr, time, time[syllables_time_stamps])]
    speech_rate_original_sr = np.zeros(len_audio_samples)
    speech_rate_original_sr[syllables_time_stamps] = 1
    speech_rate = np.zeros(len_feature_samples)
    for i in range(len_feature_samples - speech_rate_window_length_feature_samples + 1):
        speech_rate[i] = np.sum(speech_rate_original_sr[math.floor(i*feature_hop_length_samples):math.floor((i*feature_hop_length_samples)+speech_rate_window_length_samples)]) / speech_rate_window_length
    speech_rate[-speech_rate_window_length_feature_samples+1:] = speech_rate[-speech_rate_window_length_feature_samples]
    medfilt_kernel_length = config_parser.getfloat('SPEECH_RATE', 'medfilt_kernel_length')
    medfilt_kernel_length_feature_samples = math.floor(medfilt_kernel_length * feature_sample_rate) + 1 - (math.floor(medfilt_kernel_length * feature_sample_rate)%2)
    speech_rate = signal.medfilt(speech_rate, kernel_size=medfilt_kernel_length_feature_samples)
    # TODO controlla che vada bene usare quella come ultima finestra e non quella del medfilt
    speech_rate[-speech_rate_window_length_feature_samples+1:] = speech_rate[-speech_rate_window_length_feature_samples]
    # TODO vedere se passare speech rate originale o con silenzi azzerati
    speech_rate = speech_rate * voiced_filter
    config_section = config_parser['SPEECH_RATE']
    time = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    for speaker_filter in speaker_filters:
        speaker_time_stamps = feature_utils.get_speaker_time_stamps(speaker_filter, time, time)
        if speaker_time_stamps[0].shape[0] > 1:
            speech_rate[speaker_time_stamps] = feature_utils.standardize_data_wrapper(speech_rate[speaker_time_stamps], config_section)
    speech_rate = speech_rate * voiced_filter
    return speech_rate

def get_time_percent(len_audio_samples, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    percent = np.linspace(0, len_audio_samples / audio_sample_rate, len_feature_samples)
    for i in range(percent.shape[0]):
        percent[i] = percent[i] / percent[-1]
    return percent

def get_word_embeddings(transcription_and_diarization_fn, len_audio_samples, config_parser):
    audio_sample_rate = config_parser.getint('GLOBAL', 'audio_sample_rate')
    feature_window_length = config_parser.getfloat('GLOBAL', 'feature_window_length')
    feature_hop_length = config_parser.getfloat('GLOBAL', 'feature_hop_length')
    feature_window_length_samples = math.floor(feature_window_length * audio_sample_rate)
    feature_hop_length_samples = math.floor(feature_hop_length * audio_sample_rate)
    len_feature_samples = math.floor((len_audio_samples - (feature_window_length_samples - feature_hop_length_samples)) / feature_hop_length_samples)
    transcription_and_diarization_df = pd.read_csv(transcription_and_diarization_fn, dtype=transcription_and_diarization_dtype)
    model_path = config_parser.get('WORD_EMBEDDINGS', 'model_path')
    model = Word2Vec.load(model_path)
    words = []
    for word, t_st, t_end in transcription_and_diarization_df[['word', 'start_time', 'end_time']].values:
        t_st_feature_samples = math.floor((t_st * audio_sample_rate) / feature_hop_length_samples)
        t_end_feature_samples = math.floor((t_end * audio_sample_rate) / feature_hop_length_samples)
        try:
            words.append([model.wv[word], t_st_feature_samples, t_end_feature_samples])
        except KeyError:
            pass
    words = np.array(words)
    word_vector_dim = config_parser.getint('WORD_EMBEDDINGS', 'word_vector_dim')
    words_cont = np.zeros((len_feature_samples, word_vector_dim))
    for word_ts in words:
        words_cont[word_ts[1]:word_ts[2]] = word_ts[0]
    return (words, words_cont)


from typing import List

from sklearn.tree import DecisionTreeClassifier
import csv
traindata = []
testdata = []
_traindata = []
_testdata = []
buying_price = {'low': 1, 'med': 2, 'high': 3, 'vhigh': 4}
maintenance_cost = {'low': 1, 'med': 2, 'high': 3, 'vhigh': 4}
number_doors = {'2': 1, '3': 2, '4': 3, '5more': 4}
number_persons = {'2': 1, '4': 2, 'more': 3}
lug_boot = {'small': 1, 'med': 2, 'big': 3}
safety = {'low': 1, 'med': 2, 'high': 3}
trainfile = "./CarEvaluateTrain.csv"
testfile = "./CarEvaluateTest.csv"
subfile = "./submission.csv"
def trans(data,i) :
    array = []
    array.append(buying_price[data[i][0]])
    array.append(maintenance_cost[data[i][1]])
    array.append(number_doors[data[i][2]])
    array.append(number_persons[data[i][3]])
    array.append(lug_boot[data[i][4]])
    array.append(safety[data[i][5]])
    return array

def readdata():
    with open (trainfile,"r") as f:
        reader = csv.reader(f)
        rdata = [row for row in reader]
        print(rdata)
        for i in range(1,1211):
            traindata.append(trans(rdata,i))
            testdata.append(rdata[i][6])
        print(traindata)
        print(testdata)
        with open (testfile,"r") as _f:
            _reader = csv.reader(_f)
            _rdata = [row for row in _reader]
            for i in range (1,519):
                _traindata.append(trans(_rdata,i))
            print(_traindata)
            print(len(_traindata))

if __name__ == '__main__':
    readdata()
    make = DecisionTreeClassifier()
    make.fit(traindata,testdata)
    _testdata = make.predict(_traindata)
    print(_testdata)
    with open(subfile,"w",newline="") as f:
        writer = csv.writer(f)
        for i in range(len(_testdata)):
            writer.writerow([_testdata[i]])
    # array = trans(1)
    # print(array)


import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, precision_score, recall_score
import models
import process_data
import analysis

#setup
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device_count = torch.cuda.device_count()

def transfer_model(pretrained):
    if freeze_params:
        for param in pretrained.parameters():
            param.requires_grad = False
    pretrained.fc1 = nn.Linear(64,32)
    pretrained.fc2 = nn.Linear(32, 1)
    if device_count > 1:
        pretrained = nn.DataParallel(pretrained)
    pretrained = pretrained.to(device)
    return pretrained

def eval(model, val_loader, criterion):
    model.eval()
    with torch.no_grad():
        val_loss = 0
        y_pred = []
        y_true = []
        for x, y in val_loader:
            x=x.to(device).reshape(-1, 1, 187).float()
            y = y.to(device).long()
            y_hat = model(x)
            y_hat = torch.sigmoid(y_hat).reshape(-1)
            y = y.float()
            val_loss += criterion(y_hat, y).item()

            y_hat = y_hat > 0.5
            y_pred.extend(y_hat.cpu().numpy())
            y_true.extend(y.long().cpu().numpy())

        val_loss = val_loss / len(val_loader)
        f = f1_score(y_pred=y_pred, y_true=y_true)
        p = precision_score(y_pred=y_pred, y_true=y_true)
        r = recall_score(y_pred=y_pred, y_true=y_true)
        acc = accuracy_score(y_pred=y_pred, y_true=y_true)
        cm = confusion_matrix(y_true, y_pred)
        return p,r,f, acc, cm

def train(model, train_loader, val_loader, n_epochs, criterion, optimizer):
    max_f1 = 0
    for epoch in range(n_epochs):
        model.train()
        train_loss = 0
        i=0
        for x, y in train_loader:
            x=x.to(device).reshape(-1, 1, 187).float()
            y = y.to(device).long()
            y_hat = model(x)
            y_hat = torch.sigmoid(y_hat).reshape(-1)
            y = y.float()
            loss = criterion(y_hat, y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
        train_loss = train_loss / len(train_loader)
        p,r,f, acc, cm = eval(model, val_loader, criterion)
        if epoch > 10 and f > max_f1:
            max_f1=f
            print('Epoch: {} \t Training Loss: {:.6f}'.format(epoch+1, train_loss))
            print('Epoch: %d \t Validation acc: %.4f'%(epoch+1, acc))
            print('P,r,f')
            print(p,r,f)
            print(cm)

def run_mi(model_name, smote=True,batch_size=256, learning_rate=0.001, num_epochs=25, transfer='', saved_loader=True, save_path=None):
    if smote and saved_loader:
        train_loader = torch.load("train_loadermi_smote")
        val_loader = torch.load("val_loadermi_smote")
    else:
        train_loader, val_loader = process_data.load_data_mi(batch_size=batch_size, smote=smote)
    model = models.get_model(model_name, mi=True, transfer=transfer)
    criterion = nn.BCELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    train(model, train_loader, val_loader, num_epochs, criterion, optimizer)
    if save_path:
        torch.save(model.state_dict(), save_path)
    return model


import sys
import re
import pickle
import json
from pathlib import Path

# Prep.
json_file_path = sys.argv[1]
dumpfile_path = sys.argv[2]

json_file_name = json_file_path.split('/')[-1]
name_part=json_file_name.split('_')

# Create or load query_dict
if Path(dumpfile_path).is_file():
    with open(dumpfile_path, 'rb') as f:
        query_dict = pickle.load(f)
else:
    query_dict = {}

# Recunstruct query urls
if name_part[-1]=="wordSenses.json":
    word='_'.join(name_part[0:-4])
    url = 'https://babelnet.io/v6/getSenses?lemma=' + word + '&searchLang=' + name_part[-4] + '&searchLang=' + name_part[-3] + '&searchLang=' + name_part[-2] + '&searchLang=EN&key=API_KEY'

if name_part[-1]=="wordsynsets.json":
    url = 'https://babelnet.io/v6/getSynset?id=' + name_part[0] + '&targetLang=' + name_part[-4] + '&targetLang=' + name_part[-3] + '&targetLang=' + name_part[-2] + '&targetLang=EN&key=API_KEY'

if name_part[-1]=="ids.json":
    json_file_name_normalized = re.sub('_DE_ES_FR','',json_file_name)
    json_file_name_normalized = re.sub('_RU_UK_PL','',json_file_name_normalized)
    json_file_name_normalized = re.sub('_CS_IT_NN','',json_file_name_normalized)
    word = '_'.join(json_file_name_normalized.split('_')[0:-2])
    url = 'https://babelnet.io/v6/getSynsetIds?lemma=' + word + '&searchLang=EN&key=API_KEY'

# Dump query dict
if url in query_dict:
    print('Existing query in dict: ', url)
else:
    print('Adding query number ', len(query_dict)+1,' dict: ', url)
    with open(json_file_path, "r") as f:
        query_dict[url]= json.load(f)

    with open(dumpfile_path, 'wb') as f:
        pickle.dump(query_dict, f)


from os import getpid
from sys import path as pythonpath
import os
import time
from twisted.internet import reactor
from twisted.internet.threads import deferToThread

from gumby.experiments.dispersyclient import DispersyExperimentScriptClient
from time import sleep
import logging

# TODO(emilon): Fix this crap
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
pythonpath.append(os.path.abspath(os.path.join(BASE_DIR, "./tribler")))

from Tribler.Core.Session import Session

class TriblerDispersyExperimentScriptClient(DispersyExperimentScriptClient):

    def __init__(self, params):
        super(TriblerDispersyExperimentScriptClient, self).__init__(params)
        self.session = None
        self.session_config = None
        self.session_deferred = None
        self.dispersy_port = None

    def registerCallbacks(self):
        super(TriblerDispersyExperimentScriptClient, self).registerCallbacks()
        self.scenario_runner.register(self.set_dispersy_port, 'set_dispersy_port')
        self.scenario_runner.register(self.start_session, 'start_session')

    def set_dispersy_port(self, port):
        self.dispersy_port = int(port)

    def start_dispersy(self, autoload_discovery=True):
        raise NotImplementedError("Dispersy is started using the tribler session in start_session")

    def stop_dispersy(self):
        raise NotImplementedError("Dispersy is stopped using the tribler session in stop")

    def start_session(self):
        logging.error("Starting Tribler Session")
        self.session_config = self.setup_session_config()
        self.session = Session(scfg=self.session_config)

        def on_tribler_started(_):
            logging.error("Tribler Session started")
            self.annotate("Tribler Session started")
            self._dispersy = self.session.lm.dispersy

        def _do_start():
            logging.error("Upgrader")
            upgrader = self.session.prestart()
            while not upgrader.is_done:
                sleep(0.1)

            return self.session.start().addCallback(on_tribler_started)

        deferToThread(_do_start).addCallback(self.__setup_dispersy_member)

    def __setup_dispersy_member(self, _):
        self.original_on_incoming_packets = self._dispersy.on_incoming_packets
        if self.master_private_key:
            self._master_member = self._dispersy.get_member(private_key=self.master_private_key)
        else:
            self._master_member = self._dispersy.get_member(public_key=self.master_key)
        self._my_member = self.get_my_member()
        assert self._master_member
        assert self._my_member

        self._do_log()

        self.print_on_change('community-kwargs', {}, self.community_kwargs)
        self.print_on_change('community-env', {}, {'pid': getpid()})

        logging.error("Finished starting dispersy")

    def setup_session_config(self):
        from Tribler.Core.SessionConfig import SessionStartupConfig

        config = SessionStartupConfig()
        config.set_install_dir(os.path.abspath(os.path.join(BASE_DIR, "tribler")))
        config.set_state_dir(os.path.abspath(os.path.join(BASE_DIR, "output", ".Tribler-%d") % getpid()))
        config.set_torrent_checking(False)
        config.set_multicast_local_peer_discovery(False)
        config.set_megacache(False)
        config.set_dispersy(True)
        config.set_mainline_dht(True)
        config.set_torrent_collecting(False)
        config.set_libtorrent(True)
        config.set_dht_torrent_collecting(False)
        config.set_enable_torrent_search(False)
        config.set_enable_channel_search(False)
        config.set_videoplayer(False)
        config.set_listen_port(20000 + self.scenario_runner._peernumber)

        if self.dispersy_port is None:
            self.dispersy_port = 21000 + self.scenario_runner._peernumber
        config.set_dispersy_port(self.dispersy_port)
        logging.error("Dispersy port set to %d" % self.dispersy_port)
        return config

    def stop(self, retry=3):
        logging.error("Defer session stop to thread and stop reactor afterwards")
        self.annotate('end of experiment')
        return deferToThread(self.session.shutdown, False).addBoth(lambda _: reactor.callLater(10.0, reactor.stop))


from datetime import date, timedelta
from dateutil.relativedelta import relativedelta

import pytest
from reversion.models import Version
from django.core.exceptions import ValidationError
from django.template.defaultfilters import slugify

from supply_chains.models import (
    StrategicAction,
    RAGRating,
)
from supply_chains.models import SupplyChain, StrategicActionUpdate
from supply_chains.test.factories import (
    SupplyChainFactory,
    StrategicActionFactory,
    StrategicActionUpdateFactory,
)
from accounts.models import GovDepartment

pytestmark = pytest.mark.django_db

def test_supply_chain_submitted_since_query():
    """
    Tests that that the custom 'submitted_since' query method
    on the SupplyChain model returns only supply chain objects
    with a last_submission_date after the given deadline.
    """
    SupplyChainFactory.create_batch(3, last_submission_date=date(2021, 4, 7))
    SupplyChainFactory.create_batch(3, last_submission_date=date(2021, 3, 7))
    SupplyChainFactory.create_batch(3, last_submission_date=None)
    supply_chains = SupplyChain.objects.all()
    assert supply_chains.submitted_since(date(2021, 4, 1)).count() == 3

def test_archived_date_set_when_save_supply_chain():
    """Test archived_date is set when archived supply chain saved."""
    reason = "No reason"
    sc = SupplyChainFactory(is_archived=True, archived_reason=reason)
    assert sc.archived_date == date.today()
    assert sc.archived_reason == reason

def test_no_archived_date_save_strategic_action():
    """
    Test validation error is raised if an archived strategic action
    is saved without an archived_reason.
    """
    with pytest.raises(ValidationError) as excifno:
        StrategicActionFactory(is_archived=True)
    assert (
        "An archived_reason must be given when archiving a strategic action."
        in excifno.value.messages
    )

def test_no_archived_reason_save_supply_chain():
    """Validate that no supply chain can be archived w/o archive_reason"""
    msg = "An archived_reason must be given when archiving a supply chain."
    with pytest.raises(ValidationError) as excifno:
        SupplyChainFactory(is_archived=True)
        assert msg in excifno.value.messages

def test_archived_date_set_save_strategic_action():
    """Test archived_date is set when archived strategic action saved."""
    sa = StrategicActionFactory(is_archived=True, archived_reason="A reason")
    assert sa.archived_date == date.today()

def test_long_slug():
    # Arrange
    long_slug_75_chars = slugify(
        "Strategic action 1: Incentives to pivot API sourcing abdcfe asdfadsf asdfs"
    )

    # Act
    sc = SupplyChainFactory(slug=long_slug_75_chars)

    # Assert
    assert sc.slug == long_slug_75_chars


import bson
from users import get_owner
from ontologies import get_ontology, get_ontology_slug
from toxsign.assays.models import Factor, ChemicalsubFactor

def process_factors(path, user_dict, assay_dict):

    with open(path,'rb') as f:
        data = bson.decode_all(f.read())

    treated_number = 1
    fake_factors = 0
    factor_list = []
    subfactor_list = []
    onto_dict = {}
    data = sorted(data, key=lambda k:int(k['id'].split("TSF")[-1]))

    for factor in data:
        id = int(factor['id'].split("TSF")[-1])
        while not id == treated_number:
            if treated_number > id:
                raise Exception("Something went wrong when creating fake factors")
            factor_list.append(_create_fake_factor("TSF" + str(treated_number), user_dict))
            fake_factors += 1
            treated_number +=1

        factor_list.append(_create_factor(factor, user_dict, assay_dict))
        treated_number +=1

    if not (treated_number -1) == (len(data) + fake_factors):
        print(fake_factors)
        print(treated_number)
        print(len(data))
        raise Exception("Missing!")

    # Bulk create

    Factor.objects.bulk_create(factor_list)

    factor_dict = get_factor_dict()

    for factor in data:
        subfactor, onto_dict = _create_sub_factor(factor, user_dict, factor_dict, onto_dict)
        subfactor_list.append(subfactor)

    ChemicalsubFactor.objects.bulk_create(subfactor_list)
    _cleanup()

def get_factor_dict():
    dict = {}
    for factor in Factor.objects.all():
        dict[factor.tsx_id] = factor
    return dict

def get_corres_dict():
    dict = {}
    for factor in Factor.objects.all():
        if factor.assay:
            dict[factor.assay.tsx_id] = factor
    return dict

def _create_fake_factor(factor_id, user_dict):
    factor_dict = {'tsx_id': factor_id, 'created_by': user_dict['admin'], 'name': 'temp_factor_delete_me'}
    return Factor(**factor_dict)

def _cleanup():
    Factor.objects.filter(name='temp_factor_delete_me').delete()

def _create_factor(factor_data, user_dict, assay_dict):

    owner = get_owner(factor_data['owner'], user_dict)
    factor_dict = {
        'name': _get_factor_name(factor_data),
        'tsx_id':  factor_data['id'],
        'created_by': owner,
        'assay': assay_dict[factor_data['assays']]
    }

    return Factor(**factor_dict)

def _create_sub_factor(factor_data, user_dict, factor_dict, onto_dict):

    owner = get_owner(factor_data['owner'], user_dict)

    sub_fact = {
        "factor": factor_dict[factor_data['id']],
        "route": factor_data['route'],
        "vehicule" : factor_data['vehicle'],
        "exposure_time": _get_exposure_duration(factor_data),
        "exposure_frequencie": factor_data['exposure_frequencies'],
        "additional_info": factor_data['additional_information']
    }
    sub_fact["dose_value"], sub_fact["dose_unit"] =  _get_dose(factor_data)
    sub_fact["chemical"], sub_fact["chemical_slug"], onto_dict = get_ontology_slug("Chemical", factor_data, "chemical", "chemical_slug", onto_dict, "CAS")

    return ChemicalsubFactor(**sub_fact), onto_dict

def _get_factor_name(factor_data):

    li = factor_data['chemical'].split("CAS")
    if not li[0]:
        if not factor_data['chemical_name']:
            return factor_data['id']
        else:
            return factor_data['chemical_name']
    else:
        return li[0]

def _get_dose(factor_data):

    dose_unit_dict = {
        "microgkg": "µg/kg",
        "uM": "µM",
        "mg/ml": "mg/mL",
        "µM": "µM",
        "mgmL": "mg/mL",
        "mM": "mM",
        "pM": "pM",
        "mg/kg": "mg/kg",
        "fM": "fM",
        "nM": "nM",
        "microM": "µM",
        "ug/kg": "µg/kg",
        "microgmL": "µg/mL",
        "ngmL": "ng/mL"
    }

    dose = factor_data['dose']
    li = dose.split(" ")
    if len(li) > 2:
        return 0, "NA"
    if li[0] == "0":
        return  0, "NA"
    if li[0] == "IC10":
        return 10, "IC"
    else:
        return float(li[0]), dose_unit_dict[li[-1]]

def _get_exposure_duration(factor_data):

    exposure_duration = factor_data['exposure_duration']
    li = exposure_duration.split(" ")

    value = float(li[0])
    if li[-1] in ["days","day"]:
        value = value * 24
    return value


#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging

from bs4 import BeautifulSoup

from ..proxy_scrapper import ProxyScrapper

log = logging.getLogger(__name__)

class Socksproxy(ProxyScrapper):

    def __init__(self, args):
        super(Socksproxy, self).__init__(args, 'socksproxy-net')
        self.base_url = 'https://www.socks-proxy.net/'

    def scrap(self):
        self.setup_session()
        proxylist = []

        html = self.request_url(self.base_url)
        if html is None:
            log.error('Failed to download webpage: %s', self.base_url)
        else:
            log.info('Parsing proxylist from webpage: %s', self.base_url)

            soup = BeautifulSoup(html, 'html.parser')
            proxylist = self.parse_webpage(soup)

        self.session.close()
        return proxylist

    def parse_webpage(self, soup):
        proxylist = []
        counter = 0

        table = soup.find('table', attrs={'id': 'proxylisttable'})

        if not table:
            log.error('Unable to find proxylist table.')
            return proxylist

        table_rows = table.find_all('tr')
        for row in table_rows:
            columns = row.find_all('td')
            if len(columns) != 8:
                continue

            ip = columns[0].get_text().strip()
            port = columns[1].get_text().strip()
            country = columns[3].get_text().strip().lower()
            version = columns[4].get_text().strip().lower()
            status = columns[5].get_text().strip().lower()
            counter += 1

            if not self.validate_country(country):
                continue

            if version != 'socks5':
                continue

            if status == 'transparent':
                continue

            proxy_url = '{}:{}'.format(ip, port)
            proxylist.append(proxy_url)

        if self.debug and counter == 0:
            self.export_webpage(soup, self.name + '.html')

        log.info('Parsed %d socks5 proxies from webpage (found %d socks4).', len(proxylist), counter)
        return proxylist


import os
import configparser
import logging
import logging.config

class Config:
    """ Get config from config file or environment variables.
    The priority order of config is: Environment > Config file > default_value.
    The env var is composed by: [SECTION]_[OPTION]
    For example:
    get_or_else('smtp', 'BIND_ADDRESS', '127.0.0.1')
    => os.environ.get('SMTP_BIND_ADDRESS', '127.0.0.1')
    """
    config = configparser.ConfigParser()

    BOOLEAN_STATES = {'1': True, 'yes': True, 'true': True, 'on': True,
                      '0': False, 'no': False, 'false': False, 'off': False}

    @staticmethod
    def init_config(file='app/config.ini'):
        Config.config.read(file)
        logging.config.fileConfig(file, disable_existing_loggers=False)   # init logger

    @staticmethod
    def get_or_else(section, option, default_value):
        ret = os.environ.get('_'.join([section.upper(), option]))
        if ret is None:
            ret = Config.config.get(section, option, fallback=default_value)
        return ret

    @staticmethod
    def getint_or_else(section, option, default_value):
        ret = Config._get_conv_env_or_else(section, option,int,None)
        if ret is None:
            ret = Config.config.getint(section, option,fallback=default_value)
        return ret

    @staticmethod
    def getfloat_or_else(section, option, default_value):
        ret = Config._get_conv_env_or_else(section, option,float, None)
        if ret is None:
            ret = Config.config.getfloat(section, option,fallback=default_value)
        return ret

    @staticmethod
    def getboolean_or_else(section, option, default_value):
        ret = Config._get_conv_env_or_else(section, option,Config._convert_to_boolean, None)
        if ret is None:
            ret = Config.config.getboolean(section, option,fallback=default_value)
        return ret

    @staticmethod
    def _get_conv_env_or_else(section, option, conv, default_value):
        return conv(os.environ.get('_'.join([section.upper(), option.upper()]),
                                   default_value))

    @staticmethod
    def _convert_to_boolean(value):
        """Return a boolean value translating from other types if necessary.
        """
        if value.lower() not in Config.BOOLEAN_STATES:
            raise ValueError('Not a boolean: %s' % value)
        return Config.BOOLEAN_STATES[value.lower()]


#!/usr/bin/env python3

import datetime
import json
import re
import time
from argparse import ArgumentParser

import yaml
from yaml.parser import ParserError
from cassandra.cluster import Cluster

CONFIG_FILE = "config.yaml"

config = yaml.safe_load(open(CONFIG_FILE, 'r'))

config_baseURI = config['baseURI']
config_tagpacks = config['targetKeyspace']
config_header_fields = config['fields']['header']
config_tag_fields = config['fields']['tag']
config_categories = config['categories']

class TagPackException(Exception):
    pass

def extract_meta(tag_pack):
    tag_pack_meta = {k: v for k, v in tag_pack.items()
                     if k in config_header_fields and k != "tags"}
    return tag_pack_meta

def extract_tags(tag_pack):
    # Retrieve generic tag fields from tag pack header
    generic_tag_fields = {k: v for k, v in tag_pack.items()
                          if k not in config_header_fields and k != "tags"}
    # Iterate each tag and enrich them with generic fields
    for tag in tag_pack['tags']:
        final_tag = generic_tag_fields.copy()
        final_tag.update(tag)
        yield final_tag

def check_categories(d):
    for k, v in d.items():
        print(k)
        if isinstance(v, dict):
            check_categories(v)
        else:
            if k == 'category':
                if v not in config_categories:
                    return k, v
            if k == 'tags':
                for el in v:
                    if 'category' in el and el['category'] not in config_categories:
                        return k, el['category']

def lastmod_to_timestamp(d):
    for k, v in d.items():
        if isinstance(v, dict):
            d[k] = lastmod_to_timestamp(v)
        elif isinstance(v, list):
            for index, val in enumerate(v):
                v[index] = lastmod_to_timestamp(val)
        else:
            if k == 'lastmod' and isinstance(v, datetime.date):
                d[k] = int(time.mktime(v.timetuple()))
    return d

def label_to_labelnorm(label):
    # Alphanumeric and lowercase only
    pattern = re.compile(r'[\W_]+', re.UNICODE)
    return pattern.sub('', label).lower()

def verify_tag_pack(tag_pack):
    # Header should only contain header and generic body fields
    unknown_header = set(tag_pack.keys()) - \
        set(config_header_fields) - set(config_tag_fields)
    if unknown_header:
        raise TagPackException(
            'Found unknown header field: {}.'.format(unknown_header))
    # Tags should only contain body fields
    for tag in tag_pack['tags']:
        unknown_tag_field = set(tag.keys()) - set(config_tag_fields)
        if unknown_tag_field:
            raise TagPackException(
                'Found unknown tag field {} assigned with tag {}.'
                .format(unknown_tag_field, tag))
    # Check that categories are taken from defined vocabulary
    wrong_category = check_categories(tag_pack)
    if wrong_category is not None:
        raise TagPackException(
            'Found unknown category {}'.format(wrong_category))

def validate(args):
    tag_pack_files = args.tagpacks
    print(tag_pack_files)
    try:
        for tag_pack_file in tag_pack_files:
            tag_pack = yaml.safe_load(open(tag_pack_file, 'r'))
            verify_tag_pack(tag_pack)
            print("TagPack {} looks fine".format(tag_pack_file))
    except TagPackException as tagpack_error:
        print("Please check field usage:" + str(tagpack_error))
    except ParserError as parser_error:
        print("Cannot parse YAML file:" + str(parser_error))

def insert_tag_pack_meta(tag_pack_json):
    pass

def ingest(args):
    db_nodes = args.db_nodes
    if not isinstance(args.db_nodes, list):
        db_nodes = [args.db_nodes]
    cluster = Cluster(db_nodes)
    session = cluster.connect(config_tagpacks)
    session.default_timeout = 60

    tag_pack_files = args.tagpacks

    for tag_pack_file in tag_pack_files:
        tag_pack = yaml.safe_load(open(tag_pack_file, 'r'))
        tag_pack_uri = config_baseURI + '/' + tag_pack_file

        # Convert lastmod values from datetime to UNIX timestamp
        tag_pack = lastmod_to_timestamp(tag_pack)

        # Insert metadata into tagpack_by_uri table
        tag_pack_meta = extract_meta(tag_pack)
        tag_pack_meta['uri'] = tag_pack_uri
        tag_pack_meta_json = json.dumps(tag_pack_meta)
        cql_stmt = """INSERT INTO tagpack_by_uri
                      JSON '{}';""".format(tag_pack_meta_json)
        session.execute(cql_stmt)

        # Insert tags into tag_by_address table
        cql_stmt = session.prepare('INSERT INTO tag_by_address JSON ?')
        for tag in extract_tags(tag_pack):
            tag['tagpack_uri'] = tag_pack_uri
            tag_json = json.dumps(tag)
            session.execute(cql_stmt, [tag_json])

        # Insert tags into tag_by_label table
        cql_stmt = session.prepare('INSERT INTO tag_by_label JSON ?')
        for tag in extract_tags(tag_pack):
            tag['label_norm'] = label_to_labelnorm(tag['label'])
            tag['label_norm_prefix'] = tag['label_norm'][:3]
            tag['tagpack_uri'] = tag_pack_uri
            tag_json = json.dumps(tag)
            session.execute(cql_stmt, [tag_json])

        print("Ingested TagPack {}".format(tag_pack_file))

    cluster.shutdown()

def main():
    parser = ArgumentParser(description='TagPack utility',
                            epilog='GraphSense - http://graphsense.info')
    subparsers = parser.add_subparsers(title='subcommands',
                                       description='valid subcommands',
                                       help='additional help')

    # create parser for ingest command
    parser_i = subparsers.add_parser("ingest",
                                     help="Ingest TagPacks into GraphSense")
    parser_i.add_argument('-d', '--db_nodes', dest='db_nodes', nargs='+',
                          default='127.0.0.1', metavar='DB_NODE',
                          help='list of Cassandra nodes; default "localhost")')
    parser_i.set_defaults(func=ingest)

    # create parser for validate command
    parser_v = subparsers.add_parser("validate", help="Validate TagPacks")
    parser_v.set_defaults(func=validate)

    parser.add_argument("tagpacks", metavar='TAGPACK_FILE(s)',
                        type=str, nargs='+', help="TagPacks to be processed")

    args = parser.parse_args()
    args.func(args)

if __name__ == '__main__':
    t0 = time.time()
    main()
    print(time.time()-t0)