1
0
Fork 0

Add new version of algorithm

parallel_struct_est
philpMartin 4 years ago
parent e126993555
commit 0ebd040dfa
  1. 5
      main_package/classes/conditional_intensity_matrix.py
  2. 10
      main_package/classes/json_importer.py
  3. 2
      main_package/classes/network_graph.py
  4. 118
      main_package/classes/parameters_estimator.py
  5. 4
      main_package/classes/sample_path.py
  6. 4
      main_package/classes/set_of_cims.py
  7. 7
      main_package/classes/trajectory.py

@ -10,12 +10,11 @@ class ConditionalIntensityMatrix:
def update_state_transition_count(self, element_indx): def update_state_transition_count(self, element_indx):
#print(element_indx) #print(element_indx)
self.state_transition_matrix[element_indx[0]][element_indx[1]] = \ self.state_transition_matrix[element_indx[0]][element_indx[1]] += 1
self.state_transition_matrix[element_indx[0]][element_indx[1]] + 1
def update_state_residence_time_for_state(self, state, time): def update_state_residence_time_for_state(self, state, time):
#print("Time updating In state", state, time) #print("Time updating In state", state, time)
self.state_residence_times[state] = self.state_residence_times[state] + time self.state_residence_times[state] += time
def compute_cim_coefficients(self): def compute_cim_coefficients(self):
for i, row in enumerate(self.state_transition_matrix): for i, row in enumerate(self.state_transition_matrix):

@ -25,6 +25,7 @@ class JsonImporter(AbstractImporter):
self.df_samples_list = [] self.df_samples_list = []
self.df_structure = pd.DataFrame() self.df_structure = pd.DataFrame()
self.df_variables = pd.DataFrame() self.df_variables = pd.DataFrame()
self.concatenated_samples = None
super(JsonImporter, self).__init__(files_path) super(JsonImporter, self).__init__(files_path)
def import_data(self): def import_data(self):
@ -96,14 +97,23 @@ class JsonImporter(AbstractImporter):
for col_name in columns_header: for col_name in columns_header:
if col_name == 'Time': if col_name == 'Time':
sample_frame[col_name + 'Delta'] = sample_frame[col_name].diff() sample_frame[col_name + 'Delta'] = sample_frame[col_name].diff()
else:
sample_frame[col_name + 'Delta'] = (sample_frame[col_name].diff().bfill() != 0).astype(int)
sample_frame['Time'] = sample_frame['TimeDelta'] sample_frame['Time'] = sample_frame['TimeDelta']
del sample_frame['TimeDelta'] del sample_frame['TimeDelta']
sample_frame['Time'] = sample_frame['Time'].shift(-1) sample_frame['Time'] = sample_frame['Time'].shift(-1)
columns_header = list(sample_frame.columns.values)
#print(columns_header[4:])
for column in columns_header[4:]:
sample_frame[column] = sample_frame[column].shift(1)
sample_frame[column] = sample_frame[column].fillna(0)
sample_frame.drop(sample_frame.tail(1).index, inplace=True) sample_frame.drop(sample_frame.tail(1).index, inplace=True)
#print(sample_frame)
def compute_row_delta_in_all_samples_frames(self): def compute_row_delta_in_all_samples_frames(self):
for sample in self.df_samples_list: for sample in self.df_samples_list:
self.compute_row_delta_sigle_samples_frame(sample) self.compute_row_delta_sigle_samples_frame(sample)
self.concatenated_samples = pd.concat(self.df_samples_list)
def build_list_of_samples_array(self, data_frame): def build_list_of_samples_array(self, data_frame):
""" """

@ -74,7 +74,7 @@ class NetworkGraph():
indexes_for_a_node = [] indexes_for_a_node = []
for j, node in enumerate(list_of_parents): for j, node in enumerate(list_of_parents):
indexes_for_a_node.append(self.get_node_indx(node) + start_indx) indexes_for_a_node.append(self.get_node_indx(node) + start_indx)
index_structure.append(tuple(indexes_for_a_node)) index_structure.append(np.array(indexes_for_a_node, dtype=np.int))
return index_structure return index_structure
def get_nodes(self): def get_nodes(self):

@ -1,8 +1,10 @@
import os import os
import time as tm import time as tm
from line_profiler import LineProfiler from line_profiler import LineProfiler
from multiprocessing import Process
import numba as nb
import numpy as np
import network_graph as ng import network_graph as ng
import sample_path as sp import sample_path as sp
import amalgamated_cims as acims import amalgamated_cims as acims
@ -58,20 +60,73 @@ class ParametersEstimator:
self.amalgamated_cims_struct.update_state_residence_time_for_matrix( self.amalgamated_cims_struct.update_state_residence_time_for_matrix(
which_node, which_matrix, which_element, time) which_node, which_matrix, which_element, time)
def which_matrix_to_update(self, current_row, node_indx):
#print(type(self.fancy_indexing_structure[node_indx]))
return tuple(current_row.take(self.fancy_indexing_structure[node_indx]))
#return tuple(ParametersEstimator.taker(current_row, self.fancy_indexing_structure[node_indx]))
def parameters_estimation_for_variable_multiple_parents(self, node_indx, times, transitions ,variable_values, parents_values):
#print(times)
#print(variable_values)
#print(parents_values)
#print("Starting computing")
#t0 = tm.time()
for indx, row in enumerate(variable_values):
time = times[indx]
which_matrix = tuple(parents_values[indx]) # questo è un vettore
current_state = variable_values[indx]
"""if transitions[indx] == 1:
prev_state = variable_values[indx - 1]
transition = [node_indx, (prev_state, current_state)]
#which_node = transition[0]
which_element = transition[1]
self.amalgamated_cims_struct.update_state_transition_for_matrix(node_indx, which_matrix, which_element)
#which_element = current_state"""
self.amalgamated_cims_struct.update_state_residence_time_for_matrix(node_indx, which_matrix,
current_state,
time)
def find_transition(self, current_row, next_row, row_length): def parameters_estimation_for_variable_single_parent(self, node_indx, times, transitions, variable_values,
for indx in range(1, row_length): parents_values):
if current_row[indx] != next_row[indx]: for indx, row in enumerate(variable_values):
return [indx - 1, (current_row[indx], next_row[indx])] time = times[indx]
which_matrix = parents_values[indx] # Avendo un solo parent questo è uno scalare
current_state = variable_values[indx]
#which_matrix = ParametersEstimator.taker(parents_values, indx)
# print(which_matrix.dtype)
if transitions[indx] == 1:
prev_state = variable_values[indx - 1]
transition = [node_indx, (prev_state, current_state)]
which_element = transition[1]
self.amalgamated_cims_struct.update_state_transition_for_matrix(node_indx, which_matrix,
which_element)
which_element = current_state
self.amalgamated_cims_struct.update_state_residence_time_for_matrix(node_indx, which_matrix,
which_element,time)
def parameters_estimation_for_variable_no_parent(self, node_indx, times, transitions,variable_values):
def compute_time_delta(self, current_row, next_row): for indx, row in enumerate(variable_values):
return next_row[0] - current_row[0] time = times[indx]
def which_matrix_to_update(self, current_row, node_indx): which_matrix = 0
return tuple(current_row.take(self.fancy_indexing_structure[node_indx])) current_state = variable_values[indx]
"""if transitions[indx] == 1:
prev_state = variable_values[indx - 1]
#current_state = variable_values[indx]
transition = [node_indx, (prev_state, current_state)]
which_element = transition[1]
self.amalgamated_cims_struct.update_state_transition_for_matrix(node_indx, which_matrix,
which_element)"""
which_element = current_state
self.amalgamated_cims_struct.update_state_residence_time_for_matrix(node_indx, which_matrix,
which_element,
time)
#t1 = tm.time() - t0
#print("Elapsed Time ", t1)
@ -92,17 +147,46 @@ pe.init_amalgamated_cims_struct()
print(pe.amalgamated_cims_struct.get_set_of_cims(0).get_cims_number()) print(pe.amalgamated_cims_struct.get_set_of_cims(0).get_cims_number())
print(pe.amalgamated_cims_struct.get_set_of_cims(1).get_cims_number()) print(pe.amalgamated_cims_struct.get_set_of_cims(1).get_cims_number())
print(pe.amalgamated_cims_struct.get_set_of_cims(2).get_cims_number()) print(pe.amalgamated_cims_struct.get_set_of_cims(2).get_cims_number())
print(np.shape(s1.trajectories[0].transitions)[0])
#pe.parameters_estimation_for_variable(0, pe.sample_path.trajectories[0].get_trajectory()[:, 0],
# pe.sample_path.trajectories[0].get_trajectory()[:, 1], [])
#pe.parameters_estimation_single_trajectory(pe.sample_path.trajectories[0].get_trajectory()) #pe.parameters_estimation_single_trajectory(pe.sample_path.trajectories[0].get_trajectory())
pe.parameters_estimation() #pe.parameters_estimation()
"""lp = LineProfiler() lp = LineProfiler()
lp.add_function(pe.compute_sufficient_statistics_for_row) # add additional function to profile #lp.add_function(pe.compute_sufficient_statistics_for_row) # add additional function to profile
lp_wrapper = lp(pe.parameters_estimation_single_trajectory) #lp_wrapper = lp(pe.parameters_estimation_single_trajectory)
#lp_wrapper = lp(pe.parameters_estimation) #lp_wrapper = lp(pe.parameters_estimation)
lp_wrapper(pe.sample_path.trajectories[0].get_trajectory()) #lp_wrapper(pe.sample_path.trajectories[0].get_trajectory())
lp.print_stats()""" #lp.print_stats()
#lp_wrapper = lp(pe.parameters_estimation_for_variable)
#lp_wrapper(2, pe.sample_path.trajectories[0].get_times(),
#pe.sample_path.trajectories[0].get_trajectory()[:, 2],
#pe.sample_path.trajectories[0].get_trajectory()[:, [0,1]])
"""lp_wrapper = lp(pe.parameters_estimation_for_variable_single_parent)
lp_wrapper(1, pe.sample_path.trajectories[0].get_times(),
pe.sample_path.trajectories[0].get_trajectory()[:, 1],
pe.sample_path.trajectories[0].get_trajectory()[:, 2])
lp.print_stats()
#print( pe.sample_path.trajectories[0].get_trajectory()[:, [1,2]])
for matrix in pe.amalgamated_cims_struct.get_set_of_cims(1).actual_cims: for matrix in pe.amalgamated_cims_struct.get_set_of_cims(1).actual_cims:
print(matrix.state_residence_times) print(matrix.state_residence_times)
print(matrix.state_transition_matrix) print(matrix.state_transition_matrix)
matrix.compute_cim_coefficients() matrix.compute_cim_coefficients()
print(matrix.cim) print(matrix.cim)"""
"""lp_wrapper = lp(pe.parameters_estimation_for_variable_no_parent)
lp_wrapper(0, pe.sample_path.trajectories[0].get_times(), pe.sample_path.trajectories[0].transitions[:, 0],
pe.sample_path.trajectories[0].get_trajectory()[:, 0] )
lp.print_stats()
lp_wrapper = lp(pe.parameters_estimation_for_variable_single_parent)
lp_wrapper(1, pe.sample_path.trajectories[0].get_times(), pe.sample_path.trajectories[0].transitions[:, 1],
pe.sample_path.trajectories[0].get_trajectory()[:,1], pe.sample_path.trajectories[0].get_trajectory()[:,2] )
lp.print_stats()"""
lp_wrapper = lp(pe.parameters_estimation_for_variable_multiple_parents)
lp_wrapper(2, pe.sample_path.trajectories[0].get_times(), pe.sample_path.trajectories[0].transitions[:, 2],
pe.sample_path.trajectories[0].get_trajectory()[:,2], pe.sample_path.trajectories[0].get_trajectory()[:, [0,1]] )
lp.print_stats()

@ -26,8 +26,8 @@ class SamplePath:
def build_trajectories(self): def build_trajectories(self):
self.importer.import_data() self.importer.import_data()
for traj_data_frame in self.importer.df_samples_list: #for traj_data_frame in self.importer.df_samples_list:
trajectory = tr.Trajectory(self.importer.build_list_of_samples_array(traj_data_frame)) trajectory = tr.Trajectory(self.importer.build_list_of_samples_array(self.importer.concatenated_samples))
self.trajectories.append(trajectory) self.trajectories.append(trajectory)
self.importer.clear_data_frames() self.importer.clear_data_frames()

@ -43,10 +43,12 @@ class SetOfCims:
def update_state_residence_time(self, which_matrix, which_element, time): def update_state_residence_time(self, which_matrix, which_element, time):
#matrix_indx = self.indexes_converter(which_matrix) #matrix_indx = self.indexes_converter(which_matrix)
#print(type(which_matrix))
if not which_matrix: if not which_matrix:
self.actual_cims[0].update_state_residence_time_for_state(which_element, time) self.actual_cims[0].update_state_residence_time_for_state(which_element, time)
else: else:
#print(type(which_matrix))
#print(self.actual_cims[(2,2)])
self.actual_cims[which_matrix].update_state_residence_time_for_state(which_element, time) self.actual_cims[which_matrix].update_state_residence_time_for_state(which_element, time)
def build_actual_cims(self, cim_structure): def build_actual_cims(self, cim_structure):

@ -15,11 +15,16 @@ class Trajectory():
""" """
def __init__(self, list_of_columns): def __init__(self, list_of_columns):
self.actual_trajectory = np.array(list_of_columns, dtype=object).T self.actual_trajectory = np.array(list_of_columns[1:], dtype=np.int).T
self.transitions = np.array(list_of_columns[4:], dtype=np.int).T
self.times = np.array(list_of_columns[0], dtype=np.float)
def get_trajectory(self): def get_trajectory(self):
return self.actual_trajectory return self.actual_trajectory
def get_times(self):
return self.times
def size(self): def size(self):
return self.actual_trajectory.shape[0] return self.actual_trajectory.shape[0]