unimib-thesis-mad
/
PyCTBN
Archived
1
0
Fork 0
Code Releases Activity

Modify cims indexing in structure estimation

Browse Source
parallel_struct_est
philpMartin 4 years ago
parent da3533007c
commit 864ffbd319
8 changed files with 167 additions and 66 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 26
      main_package/classes/json_importer.py
  2. 56
      main_package/classes/network_graph.py
  3. 18
      main_package/classes/parameters_estimator.py
  4. 1
      main_package/classes/sample_path.py
  5. 11
      main_package/classes/set_of_cims.py
  6. 8
      main_package/classes/sets_of_cims_container.py
  7. 94
      main_package/classes/structure_estimator.py
  8. 19
      main_package/tests/test_structure.py

26
main_package/classes/json_importer.py
Unescape View File

@ -39,12 +39,12 @@ class JsonImporter(AbstractImporter):
def import_data(self): def import_data(self):
raw_data = self.read_json_file() raw_data = self.read_json_file()
self.import_variables(raw_data) #self.import_variables(raw_data)
self.import_trajectories(raw_data) self.import_trajectories(raw_data)
self.compute_row_delta_in_all_samples_frames(self.time_key) self.compute_row_delta_in_all_samples_frames(self.time_key)
self.clear_data_frame_list() self.clear_data_frame_list()
self.import_structure(raw_data) self.import_structure(raw_data)
#self.import_variables(raw_data, self.sorter) self.import_variables(raw_data, self.sorter)
def import_trajectories(self, raw_data: pd.DataFrame): def import_trajectories(self, raw_data: pd.DataFrame):
self.normalize_trajectories(raw_data, 0, self.samples_label) self.normalize_trajectories(raw_data, 0, self.samples_label)
@ -53,14 +53,16 @@ class JsonImporter(AbstractImporter):
self._df_structure = self.one_level_normalizing(raw_data, 0, self.structure_label) self._df_structure = self.one_level_normalizing(raw_data, 0, self.structure_label)
#TODO Attenzione l'ordine delle vars non è alfabetico come nel dataset -> agire di conseguenza #TODO Attenzione l'ordine delle vars non è alfabetico come nel dataset -> agire di conseguenza
#Ordinando la vars alfabeticamente #Ordinando la vars alfabeticamente
def import_variables(self, raw_data: pd.DataFrame): def import_variables(self, raw_data: pd.DataFrame, sorter):
self._df_variables = self.one_level_normalizing(raw_data, 0, self.variables_label) self._df_variables = self.one_level_normalizing(raw_data, 0, self.variables_label)
self.sorter = self._df_variables[self.variables_key].to_list() #self.sorter = self._df_variables[self.variables_key].to_list()
self.sorter.sort() #self.sorter.sort()
print("Sorter:", self.sorter) #print("Sorter:", self.sorter)
self._df_variables[self.variables_key] = self._df_variables[self.variables_key].astype("category") self._df_variables[self.variables_key] = self._df_variables[self.variables_key].astype("category")
self._df_variables[self.variables_key] = self._df_variables[self.variables_key].cat.set_categories(self.sorter) self._df_variables[self.variables_key] = self._df_variables[self.variables_key].cat.set_categories(sorter)
self._df_variables = self._df_variables.sort_values([self.variables_key]) self._df_variables = self._df_variables.sort_values([self.variables_key])
self._df_variables.reset_index(inplace=True)
#print("Var Frame", self._df_variables)
def read_json_file(self) -> typing.List: def read_json_file(self) -> typing.List:
""" """
@ -124,8 +126,9 @@ class JsonImporter(AbstractImporter):
return sample_frame return sample_frame
def compute_row_delta_in_all_samples_frames(self, time_header_label: str): def compute_row_delta_in_all_samples_frames(self, time_header_label: str):
#columns_header = list(self.df_samples_list[0].columns.values) columns_header = list(self.df_samples_list[0].columns.values)
#self.sorter = columns_header[1:] columns_header.remove('Time')
self.sorter = columns_header
shifted_cols_header = [s + "S" for s in self.sorter] shifted_cols_header = [s + "S" for s in self.sorter]
compute_row_delta = self.compute_row_delta_sigle_samples_frame compute_row_delta = self.compute_row_delta_sigle_samples_frame
"""for indx, sample in enumerate(self.df_samples_list): """for indx, sample in enumerate(self.df_samples_list):
@ -133,13 +136,12 @@ class JsonImporter(AbstractImporter):
time_header_label, self.sorter, shifted_cols_header)""" time_header_label, self.sorter, shifted_cols_header)"""
self.df_samples_list = [compute_row_delta(sample, time_header_label, self.sorter, shifted_cols_header) for sample in self.df_samples_list] self.df_samples_list = [compute_row_delta(sample, time_header_label, self.sorter, shifted_cols_header) for sample in self.df_samples_list]
self._concatenated_samples = pd.concat(self.df_samples_list) self._concatenated_samples = pd.concat(self.df_samples_list)
#TODO Attenzione la colonna di indice 0 non è sempre quella del tempo ordinare il daframe concatenato di conseguenza
complete_header = self.sorter[:] complete_header = self.sorter[:]
complete_header.insert(0,'Time') complete_header.insert(0,'Time')
complete_header.extend(shifted_cols_header) complete_header.extend(shifted_cols_header)
print("Complete Header", complete_header) #print("Complete Header", complete_header)
self._concatenated_samples = self._concatenated_samples[complete_header] self._concatenated_samples = self._concatenated_samples[complete_header]
print("Concat Samples",self._concatenated_samples) #print("Concat Samples",self._concatenated_samples)
def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List: def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List:
""" """

56
main_package/classes/network_graph.py
Unescape View File

@ -19,29 +19,31 @@ class NetworkGraph():
self.graph = nx.DiGraph() self.graph = nx.DiGraph()
self._nodes_indexes = self.graph_struct.list_of_nodes_indexes() self._nodes_indexes = self.graph_struct.list_of_nodes_indexes()
self._nodes_labels = self.graph_struct.list_of_nodes_labels() self._nodes_labels = self.graph_struct.list_of_nodes_labels()
self._nodes_values = self.graph_struct.nodes_values()
self.aggregated_info_about_nodes_parents = None self.aggregated_info_about_nodes_parents = None
self._fancy_indexing = None self._fancy_indexing = None
self._time_scalar_indexing_structure = None self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None self._transition_scalar_indexing_structure = None
self._time_filtering = None self._time_filtering = None
self._transition_filtering = None self._transition_filtering = None
self._p_combs_structure = None
def init_graph(self): def init_graph(self):
self.add_nodes(self.graph_struct.list_of_nodes_labels()) self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.list_of_edges()) self.add_edges(self.graph_struct.list_of_edges())
self.aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes() self.aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0) self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures() self.build_scalar_indexing_structures()
#self.build_time_scalar_indexing_structure()
self.build_time_columns_filtering_structure() self.build_time_columns_filtering_structure()
#self.build_transition_scalar_indexing_structure()
self.build_transition_columns_filtering_structure() self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
#ATTENZIONE LIST_OF_NODES DEVE ESSERE COERENTE CON L?ORDINAMENTO DEL DS
def add_nodes(self, list_of_nodes): def add_nodes(self, list_of_nodes):
#self.graph.add_nodes_from(list_of_nodes) #self.graph.add_nodes_from(list_of_nodes)
nodes_indxs = self.graph_struct.list_of_nodes_indexes() nodes_indxs = self._nodes_indexes
nodes_vals = self.graph_struct.nodes_values() nodes_vals = self.graph_struct.nodes_values()
pos = 0 pos = 0
#print("LIST OF NODES", list_of_nodes)
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals): for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self.graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos) self.graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1 pos += 1
@ -52,13 +54,21 @@ class NetworkGraph():
def get_ordered_by_indx_set_of_parents(self, node): def get_ordered_by_indx_set_of_parents(self, node):
parents = self.get_parents_by_id(node) parents = self.get_parents_by_id(node)
#print("PARENTS", parents)
nodes = self.get_nodes() nodes = self.get_nodes()
sorted_parents = [x for _, x in sorted(zip(nodes, parents))] #print("NODES", nodes)
d = {v: i for i, v in enumerate(nodes)}
sorted_parents = sorted(parents, key=lambda v: d[v])
#sorted_parents = [x for _, x in sorted(zip(nodes, parents))]
#print("SORTED PARENTS IN GRAPH",sorted_parents)
#p_indxes= [] #p_indxes= []
#p_values = [] #p_values = []
get_node_indx = self.get_node_indx get_node_indx = self.get_node_indx
p_indxes = [get_node_indx(node) for node in sorted_parents] p_indxes = [get_node_indx(node) for node in sorted_parents]
#p_indxes.sort()
p_values = [self.get_states_number(node) for node in sorted_parents] p_values = [self.get_states_number(node) for node in sorted_parents]
#print("P INDXS", p_indxes)
#print("P VALS", p_values)
return (sorted_parents, p_indxes, p_values) return (sorted_parents, p_indxes, p_values)
def get_ord_set_of_par_of_all_nodes(self): def get_ord_set_of_par_of_all_nodes(self):
@ -88,9 +98,9 @@ class NetworkGraph():
#states_number_list = [] #states_number_list = []
#for node in self._nodes_labels: #for node in self._nodes_labels:
#states_number_list.append(self.get_states_number(node)) #states_number_list.append(self.get_states_number(node))
get_states_number = self.get_states_number #get_states_number = self.get_states_number
states_number_list = [get_states_number(node) for node in self._nodes_labels] #states_number_list = [get_states_number(node) for node in self._nodes_labels]
return states_number_list return self._nodes_values
def build_fancy_indexing_structure(self, start_indx): def build_fancy_indexing_structure(self, start_indx):
if start_indx > 0: if start_indx > 0:
@ -125,6 +135,8 @@ class NetworkGraph():
"""for node_indx, p_indxs in zip(self.graph_struct.list_of_nodes_indexes(), self._fancy_indexing): """for node_indx, p_indxs in zip(self.graph_struct.list_of_nodes_indexes(), self._fancy_indexing):
self._time_filtering.append(np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int))""" self._time_filtering.append(np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int))"""
nodes_indxs = self.graph_struct.list_of_nodes_indexes() nodes_indxs = self.graph_struct.list_of_nodes_indexes()
#print("FINDXING", self._fancy_indexing)
#print("Nodes Indxs", nodes_indxs)
self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int) self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)] for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)]
@ -150,6 +162,28 @@ class NetworkGraph():
self._transition_scalar_indexing_structure = [i[0] for i in aggr] self._transition_scalar_indexing_structure = [i[0] for i in aggr]
self._time_scalar_indexing_structure = [i[1] for i in aggr] self._time_scalar_indexing_structure = [i[1] for i in aggr]
def build_p_combs_structure(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs_struct = [self.build_p_comb_structure_for_a_node(p_vals) for p_vals in parents_values_for_all_nodes]
return p_combs_struct
def build_p_comb_structure_for_a_node(self, parents_values):
tmp = []
for val in parents_values:
tmp.append([x for x in range(val)])
#print("TIMP", tmp)
if len(parents_values) > 0:
parents_comb = np.array(np.meshgrid(*tmp)).T.reshape(-1, len(parents_values))
#print("PArents COmb", parents_comb)
if len(parents_values) > 1:
tmp_comb = parents_comb[:, 1].copy()
#print(tmp_comb)
parents_comb[:, 1] = parents_comb[:, 0].copy()
parents_comb[:, 0] = tmp_comb
else:
parents_comb = np.array([[]], dtype=np.int)
return parents_comb
def get_nodes(self): def get_nodes(self):
return list(self.graph.nodes) return list(self.graph.nodes)
@ -195,4 +229,8 @@ class NetworkGraph():
def transition_filtering(self): def transition_filtering(self):
return self._transition_filtering return self._transition_filtering
@property
def p_combs(self):
return self._p_combs_structure

18
main_package/classes/parameters_estimator.py
Unescape View File

@ -19,7 +19,8 @@ class ParametersEstimator:
def init_sets_cims_container(self): def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self.net_graph.get_nodes(), self.sets_of_cims_struct = acims.SetsOfCimsContainer(self.net_graph.get_nodes(),
self.net_graph.get_states_number_of_all_nodes_sorted(), self.net_graph.get_states_number_of_all_nodes_sorted(),
self.net_graph.get_ordered_by_indx_parents_values_for_all_nodes()) self.net_graph.get_ordered_by_indx_parents_values_for_all_nodes(),
self.net_graph.p_combs)
def compute_parameters(self): def compute_parameters(self):
@ -42,26 +43,28 @@ class ParametersEstimator:
self.net_graph.transition_scalar_indexing_structure[indx], self.net_graph.transition_scalar_indexing_structure[indx],
aggr[1].transition_matrices) aggr[1].transition_matrices)
aggr[1].build_cims(aggr[1].state_residence_times, aggr[1].transition_matrices) aggr[1].build_cims(aggr[1].state_residence_times, aggr[1].transition_matrices)
#TODO togliere tutti sti self.
def compute_parameters_for_node(self, node_id): def compute_parameters_for_node(self, node_id):
pos_index = self.net_graph.graph_struct.get_positional_node_indx(node_id) pos_index = self.net_graph.get_positional_node_indx(node_id)
node_indx = self.net_graph.get_node_indx(node_id) node_indx = self.net_graph.get_node_indx(node_id)
state_res_times = self.sets_of_cims_struct.sets_of_cims[pos_index].state_residence_times
transition_matrices = self.sets_of_cims_struct.sets_of_cims[pos_index].transition_matrices
#print("Nodes", self.net_graph.get_nodes()) #print("Nodes", self.net_graph.get_nodes())
self.compute_state_res_time_for_node(node_indx, self.sample_path.trajectories.times, self.compute_state_res_time_for_node(node_indx, self.sample_path.trajectories.times,
self.sample_path.trajectories.trajectory, self.sample_path.trajectories.trajectory,
self.net_graph.time_filtering[pos_index], self.net_graph.time_filtering[pos_index],
self.net_graph.time_scalar_indexing_strucure[pos_index], self.net_graph.time_scalar_indexing_strucure[pos_index],
self.sets_of_cims_struct.sets_of_cims[pos_index].state_residence_times) state_res_times)
# print(self.net_graph.transition_filtering[indx]) # print(self.net_graph.transition_filtering[indx])
# print(self.net_graph.transition_scalar_indexing_structure[indx]) # print(self.net_graph.transition_scalar_indexing_structure[indx])
self.compute_state_transitions_for_a_node(node_indx, self.compute_state_transitions_for_a_node(node_indx,
self.sample_path.trajectories.complete_trajectory, self.sample_path.trajectories.complete_trajectory,
self.net_graph.transition_filtering[pos_index], self.net_graph.transition_filtering[pos_index],
self.net_graph.transition_scalar_indexing_structure[pos_index], self.net_graph.transition_scalar_indexing_structure[pos_index],
self.sets_of_cims_struct.sets_of_cims[pos_index].transition_matrices) transition_matrices)
self.sets_of_cims_struct.sets_of_cims[pos_index].build_cims( self.sets_of_cims_struct.sets_of_cims[pos_index].build_cims(
self.sets_of_cims_struct.sets_of_cims[pos_index].state_residence_times, state_res_times,
self.sets_of_cims_struct.sets_of_cims[pos_index].transition_matrices) transition_matrices)
def compute_state_res_time_for_node(self, node_indx, times, trajectory, cols_filter, scalar_indexes_struct, T): def compute_state_res_time_for_node(self, node_indx, times, trajectory, cols_filter, scalar_indexes_struct, T):
#print(times.size) #print(times.size)
@ -98,6 +101,7 @@ class ParametersEstimator:
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int), M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int),
minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2]) minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
#print(M)
M_raveled = M.ravel() M_raveled = M.ravel()
M_raveled[diag_indices] = 0 M_raveled[diag_indices] = 0
#print(M_raveled) #print(M_raveled)

1
main_package/classes/sample_path.py
Unescape View File

@ -34,6 +34,7 @@ class SamplePath:
def build_structure(self): def build_structure(self):
self.total_variables_count = len(self.importer.sorter) self.total_variables_count = len(self.importer.sorter)
labels = self.importer._df_variables['Name'].to_list() labels = self.importer._df_variables['Name'].to_list()
#print("SAMPLE PATH LABELS",labels)
indxs = self.importer._df_variables.index.to_numpy() indxs = self.importer._df_variables.index.to_numpy()
vals = self.importer._df_variables['Value'].to_numpy() vals = self.importer._df_variables['Value'].to_numpy()
edges = list(self.importer._df_structure.to_records(index=False)) edges = list(self.importer._df_structure.to_records(index=False))

11
main_package/classes/set_of_cims.py
Unescape View File

@ -12,13 +12,14 @@ class SetOfCims:
:actual_cims: le CIM della varibile :actual_cims: le CIM della varibile
""" """
def __init__(self, node_id, parents_states_number, node_states_number): def __init__(self, node_id, parents_states_number, node_states_number, p_combs):
self.node_id = node_id self.node_id = node_id
self.parents_states_number = parents_states_number self.parents_states_number = parents_states_number
self.node_states_number = node_states_number self.node_states_number = node_states_number
self.actual_cims = [] self.actual_cims = []
self.state_residence_times = None self.state_residence_times = None
self.transition_matrices = None self.transition_matrices = None
self.p_combs = p_combs
self.build_actual_cims_structure() self.build_actual_cims_structure()
def build_actual_cims_structure(self): def build_actual_cims_structure(self):
@ -52,6 +53,7 @@ class SetOfCims:
cim_to_add.compute_cim_coefficients() cim_to_add.compute_cim_coefficients()
#print(cim_to_add) #print(cim_to_add)
self.actual_cims.append(cim_to_add) self.actual_cims.append(cim_to_add)
self.actual_cims = np.array(self.actual_cims)
self.transition_matrices = None self.transition_matrices = None
self.state_residence_times = None self.state_residence_times = None
@ -62,6 +64,13 @@ class SetOfCims:
flat_index = self.indexes_converter(index) flat_index = self.indexes_converter(index)
return self.actual_cims[flat_index] return self.actual_cims[flat_index]
def filter_cims_with_mask(self, mask_arr, comb):
if mask_arr.size <= 1:
return self.actual_cims
else:
tmp_parents_comb_from_ids = np.argwhere(np.all(self.p_combs[:, mask_arr] == comb, axis=1)).ravel()
#print("CIMS INDEXES TO USE!",tmp_parents_comb_from_ids)
return self.actual_cims[tmp_parents_comb_from_ids]
"""sofc = SetOfCims('Z', [3, 3], 3) """sofc = SetOfCims('Z', [3, 3], 3)
sofc.build_actual_cims_structure() sofc.build_actual_cims_structure()

8
main_package/classes/sets_of_cims_container.py
Unescape View File

@ -6,16 +6,16 @@ class SetsOfCimsContainer:
""" """
Aggrega un insieme di oggetti SetOfCims Aggrega un insieme di oggetti SetOfCims
""" """
def __init__(self, list_of_keys, states_number_per_node, list_of_parents_states_number): def __init__(self, list_of_keys, states_number_per_node, list_of_parents_states_number, p_combs_list):
self.sets_of_cims = None self.sets_of_cims = None
self.init_cims_structure(list_of_keys, states_number_per_node, list_of_parents_states_number) self.init_cims_structure(list_of_keys, states_number_per_node, list_of_parents_states_number, p_combs_list)
#self.states_per_variable = states_number #self.states_per_variable = states_number
def init_cims_structure(self, keys, states_number_per_node, list_of_parents_states_number): def init_cims_structure(self, keys, states_number_per_node, list_of_parents_states_number, p_combs_list):
"""for indx, key in enumerate(keys): """for indx, key in enumerate(keys):
self.sets_of_cims.append( self.sets_of_cims.append(
socim.SetOfCims(key, list_of_parents_states_number[indx], states_number_per_node[indx]))""" socim.SetOfCims(key, list_of_parents_states_number[indx], states_number_per_node[indx]))"""
self.sets_of_cims = [socim.SetOfCims(pair[1], list_of_parents_states_number[pair[0]], states_number_per_node[pair[0]]) self.sets_of_cims = [socim.SetOfCims(pair[1], list_of_parents_states_number[pair[0]], states_number_per_node[pair[0]], p_combs_list[pair[0]])
for pair in enumerate(keys)] for pair in enumerate(keys)]
def get_set_of_cims(self, node_indx): def get_set_of_cims(self, node_indx):

94
main_package/classes/structure_estimator.py
Unescape View File

@ -16,8 +16,11 @@ class StructureEstimator:
def __init__(self, sample_path, exp_test_alfa, chi_test_alfa): def __init__(self, sample_path, exp_test_alfa, chi_test_alfa):
self.sample_path = sample_path self.sample_path = sample_path
self.nodes = np.array(self.sample_path.structure.list_of_nodes_labels()) self.nodes = np.array(self.sample_path.structure.list_of_nodes_labels())
#print("NODES", self.nodes)
self.nodes_vals = self.sample_path.structure.nodes_vals_arr self.nodes_vals = self.sample_path.structure.nodes_vals_arr
self.nodes_indxs = self.sample_path.structure.nodes_indexes_arr self.nodes_indxs = self.sample_path.structure.nodes_indexes_arr
#self.nodes_indxs = np.array(range(0,4))
#print("INDXS", self.nodes_indxs)
self.complete_graph = self.build_complete_graph(self.sample_path.structure.list_of_nodes_labels()) self.complete_graph = self.build_complete_graph(self.sample_path.structure.list_of_nodes_labels())
self.exp_test_sign = exp_test_alfa self.exp_test_sign = exp_test_alfa
self.chi_test_alfa = chi_test_alfa self.chi_test_alfa = chi_test_alfa
@ -35,9 +38,20 @@ class StructureEstimator:
return complete_graph return complete_graph
def complete_test(self, test_parent, test_child, parent_set, child_states_numb, tot_vars_count): def complete_test(self, test_parent, test_child, parent_set, child_states_numb, tot_vars_count):
#print("Test Parent:", test_parent)
#print("Sep Set", parent_set)
p_set = parent_set[:] p_set = parent_set[:]
complete_info = parent_set[:] complete_info = parent_set[:]
complete_info.append(test_child) complete_info.append(test_child)
parents = np.array(parent_set)
parents = np.append(parents, test_parent)
#print("PARENTS", parents)
#parents.sort()
sorted_parents = self.nodes[np.isin(self.nodes, parents)]
#print("SORTED PARENTS", sorted_parents)
cims_filter = sorted_parents != test_parent
#print("PARENTS NO FROM MASK", cims_filter)
if not p_set: if not p_set:
sofc1 = self.cache.find(test_child) sofc1 = self.cache.find(test_child)
else: else:
@ -45,15 +59,22 @@ class StructureEstimator:
if not sofc1: if not sofc1:
bool_mask1 = np.isin(self.nodes,complete_info) bool_mask1 = np.isin(self.nodes,complete_info)
#print("Bool mask 1", bool_mask1)
l1 = list(self.nodes[bool_mask1]) l1 = list(self.nodes[bool_mask1])
#print("L1", l1)
indxs1 = self.nodes_indxs[bool_mask1] indxs1 = self.nodes_indxs[bool_mask1]
#print("INDXS 1", indxs1)
vals1 = self.nodes_vals[bool_mask1] vals1 = self.nodes_vals[bool_mask1]
eds1 = list(itertools.product(parent_set,test_child)) eds1 = list(itertools.product(parent_set,test_child))
s1 = st.Structure(l1, indxs1, vals1, eds1, tot_vars_count) s1 = st.Structure(l1, indxs1, vals1, eds1, tot_vars_count)
g1 = ng.NetworkGraph(s1) g1 = ng.NetworkGraph(s1)
g1.init_graph() g1.init_graph()
print("G1 NODES", g1.get_nodes()) #print("M Vector", g1.transition_scalar_indexing_structure)
print("G1 Edges", g1.get_edges()) #print("Time Vecotr", g1.time_scalar_indexing_strucure)
#print("Time Filter", g1.time_filtering)
#print("M Filter", g1.transition_filtering)
#print("G1 NODES", g1.get_nodes())
#print("G1 Edges", g1.get_edges())
p1 = pe.ParametersEstimator(self.sample_path, g1) p1 = pe.ParametersEstimator(self.sample_path, g1)
p1.init_sets_cims_container() p1.init_sets_cims_container()
p1.compute_parameters_for_node(test_child) p1.compute_parameters_for_node(test_child)
@ -71,7 +92,8 @@ class StructureEstimator:
#print("PSET ", p_set) #print("PSET ", p_set)
#set_p_set = set(p_set) #set_p_set = set(p_set)
sofc2 = self.cache.find(set(p_set)) sofc2 = self.cache.find(set(p_set))
#print("Sofc2 ", sofc2) #if sofc2:
#print("Sofc2 in CACHE ", sofc2.actual_cims)
#print(self.cache.list_of_sets_of_indxs) #print(self.cache.list_of_sets_of_indxs)
"""p2 = pe.ParametersEstimator(self.sample_path, g2) """p2 = pe.ParametersEstimator(self.sample_path, g2)
@ -82,32 +104,45 @@ class StructureEstimator:
if not sofc2: if not sofc2:
complete_info.append(test_parent) complete_info.append(test_parent)
bool_mask2 = np.isin(self.nodes, complete_info) bool_mask2 = np.isin(self.nodes, complete_info)
#print("BOOL MASK 2",bool_mask2)
l2 = list(self.nodes[bool_mask2]) l2 = list(self.nodes[bool_mask2])
#print("L2", l2)
indxs2 = self.nodes_indxs[bool_mask2] indxs2 = self.nodes_indxs[bool_mask2]
#print("INDXS 2", indxs2)
vals2 = self.nodes_vals[bool_mask2] vals2 = self.nodes_vals[bool_mask2]
eds2 = list(itertools.product(p_set, test_child)) eds2 = list(itertools.product(p_set, test_child))
s2 = st.Structure(l2, indxs2, vals2, eds2, tot_vars_count) s2 = st.Structure(l2, indxs2, vals2, eds2, tot_vars_count)
g2 = ng.NetworkGraph(s2) g2 = ng.NetworkGraph(s2)
g2.init_graph() g2.init_graph()
print("G2 Nodes", g2.get_nodes()) #print("M Vector", g2.transition_scalar_indexing_structure)
print("G2 Edges", g2.get_edges()) #print("Time Vecotr", g2.time_scalar_indexing_strucure)
#print("Time Filter", g2.time_filtering)
#print("M Filter", g2.transition_filtering)
#print("G2 Nodes", g2.get_nodes())
#print("G2 Edges", g2.get_edges())
p2 = pe.ParametersEstimator(self.sample_path, g2) p2 = pe.ParametersEstimator(self.sample_path, g2)
p2.init_sets_cims_container() p2.init_sets_cims_container()
p2.compute_parameters_for_node(test_child) p2.compute_parameters_for_node(test_child)
sofc2 = p2.sets_of_cims_struct.sets_of_cims[g2.get_positional_node_indx(test_child)] sofc2 = p2.sets_of_cims_struct.sets_of_cims[g2.get_positional_node_indx(test_child)]
if p_set: #if p_set:
#set_p_set = set(p_set) #set_p_set = set(p_set)
self.cache.put(set(p_set), sofc2) self.cache.put(set(p_set), sofc2)
end = 0 #start = 0
increment = self.sample_path.structure.get_states_number(test_parent) #end = self.sample_path.structure.get_states_number(test_parent)
for cim1 in sofc1.actual_cims: #print("SOFC2", sofc2.actual_cims)
start = end #print("Sofc2 pcomb", sofc2.p_combs)
end = start + increment for cim1, p_comb in zip(sofc1.actual_cims, sofc1.p_combs):
for j in range(start, end): #print("GETTING THIS P COMB", p_comb)
#if len(parent_set) > 1:
cond_cims = sofc2.filter_cims_with_mask(cims_filter, p_comb)
#else:
#cond_cims = sofc2.actual_cims
#print("COnd Cims", cond_cims)
for cim2 in cond_cims:
#cim2 = sofc2.actual_cims[j] #cim2 = sofc2.actual_cims[j]
#print(indx) #print(indx)
#print("Run Test", i, j) #print("Run Test", i, j)
if not self.independence_test(child_states_numb, cim1, sofc2.actual_cims[j]): if not self.independence_test(child_states_numb, cim1, cim2):
return False return False
return True return True
@ -118,8 +153,8 @@ class StructureEstimator:
r2s = M2.diagonal() r2s = M2.diagonal()
C1 = cim1.cim C1 = cim1.cim
C2 = cim2.cim C2 = cim2.cim
print("C1", C1) #print("C1", C1)
print("C2", C2) #print("C2", C2)
F_stats = C2.diagonal() / C1.diagonal() F_stats = C2.diagonal() / C1.diagonal()
exp_alfa = self.exp_test_sign exp_alfa = self.exp_test_sign
for val in range(0, child_states_numb): for val in range(0, child_states_numb):
@ -156,7 +191,7 @@ class StructureEstimator:
return True return True
def one_iteration_of_CTPC_algorithm(self, var_id, tot_vars_count): def one_iteration_of_CTPC_algorithm(self, var_id, tot_vars_count):
print("TESTING VAR", var_id) print("##################TESTING VAR################", var_id)
u = list(self.complete_graph.predecessors(var_id)) u = list(self.complete_graph.predecessors(var_id))
#tests_parents_numb = len(u) #tests_parents_numb = len(u)
#complete_frame = self.complete_graph_frame #complete_frame = self.complete_graph_frame
@ -167,22 +202,26 @@ class StructureEstimator:
#for parent_id in u: #for parent_id in u:
parent_indx = 0 parent_indx = 0
while parent_indx < len(u): while parent_indx < len(u):
# list_without_test_parent = u.remove(parent_id) #print("Parent_indx",parent_indx)
#print("LEN U", len(u))
removed = False removed = False
#if not list(self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])): #if not list(self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])):
#break #break
S = self.generate_possible_sub_sets_of_size(u, b, parent_indx) S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])
#print("U Set", u) #print("U Set", u)
#print("S", S) #print("S", S)
test_parent = u[parent_indx]
#print("Test Parent", test_parent)
for parents_set in S: for parents_set in S:
#print("Parent Set", parents_set) #print("Parent Set", parents_set)
#print("Test Parent", u[parent_indx]) #print("Test Parent", test_parent)
if self.complete_test(u[parent_indx], var_id, parents_set, child_states_numb, tot_vars_count): if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count):
print("Removing EDGE:", u[parent_indx], var_id) #print("Removing EDGE:", test_parent, var_id)
self.complete_graph.remove_edge(u[parent_indx], var_id) self.complete_graph.remove_edge(test_parent, var_id)
del u[parent_indx] u.remove(test_parent)
removed = True removed = True
break
#else: #else:
#parent_indx += 1 #parent_indx += 1
if not removed: if not removed:
@ -192,8 +231,11 @@ class StructureEstimator:
def generate_possible_sub_sets_of_size(self, u, size, parent_indx): def generate_possible_sub_sets_of_size(self, u, size, parent_indx):
list_without_test_parent = u[:] list_without_test_parent = u[:]
del list_without_test_parent[parent_indx] #del list_without_test_parent[parent_indx]
# u.remove(parent_id) #print("U", u)
#print("Szie", size)
#print("parent indx", parent_indx)
list_without_test_parent.remove(parent_indx)
#print(list(map(list, itertools.combinations(list_without_test_parent, size)))) #print(list(map(list, itertools.combinations(list_without_test_parent, size))))
return map(list, itertools.combinations(list_without_test_parent, size)) return map(list, itertools.combinations(list_without_test_parent, size))

19
main_package/tests/test_structure.py
Unescape View File

@ -107,17 +107,22 @@ class TestStructure(unittest.TestCase):
sp1 = sp.SamplePath('../data', 'samples', 'dyn.str', 'variables', 'Time', 'Name') sp1 = sp.SamplePath('../data', 'samples', 'dyn.str', 'variables', 'Time', 'Name')
sp1.build_trajectories() sp1.build_trajectories()
sp1.build_structure() sp1.build_structure()
st1 = st.Structure(['Y'], np.array([2]), np.array([2]), [], sp1.total_variables_count) st1 = st.Structure(['X','Y','Z'], np.array([0,1,2]), np.array([3,3,3]), [('Z','X'),('Y', 'X')], sp1.total_variables_count)
g1 = ng.NetworkGraph(st1) g1 = ng.NetworkGraph(st1)
g1.init_graph() g1.init_graph()
print(g1.transition_scalar_indexing_structure) print("M Vector",g1.transition_scalar_indexing_structure)
print(g1.time_scalar_indexing_strucure) print("Time Vecotr",g1.time_scalar_indexing_strucure)
print(g1.time_filtering) print("Time Filter",g1.time_filtering)
print(g1.transition_filtering) print("M Filter",g1.transition_filtering)
print(g1.p_combs)
print("AGG STR", g1.aggregated_info_about_nodes_parents)
p1 = pe.ParametersEstimator(sp1,g1) p1 = pe.ParametersEstimator(sp1,g1)
p1.init_sets_cims_container() p1.init_sets_cims_container()
p1.compute_parameters_for_node('Y') p1.compute_parameters_for_node('X')
print(p1.sets_of_cims_struct.sets_of_cims[0].actual_cims) #print(p1.sets_of_cims_struct.get_cims_of_node(0,[1,0]))
print(p1.sets_of_cims_struct.sets_of_cims[1].actual_cims)
#print(p1.sets_of_cims_struct.sets_of_cims[2].get_cims_where_parents_except_last_are_in_state(np.array([0])))
#print(p1.sets_of_cims_struct.sets_of_cims[0].p_combs)
if __name__ == '__main__': if __name__ == '__main__':