1
0
Fork 0

Refactor class members as protected;Refactors on comments and methods return types

parallel_struct_est
philpMartin 4 years ago
parent 59cda0d3d9
commit 5321999e5e
  1. 10
      main_package/basic_main.py
  2. 23
      main_package/classes/abstract_importer.py
  3. 26
      main_package/classes/cache.py
  4. 11
      main_package/classes/conditional_intensity_matrix.py
  5. 88
      main_package/classes/json_importer.py
  6. 198
      main_package/classes/network_graph.py
  7. 106
      main_package/classes/parameters_estimator.py
  8. 38
      main_package/classes/sample_path.py
  9. 110
      main_package/classes/set_of_cims.py
  10. 2
      main_package/classes/simple_cvs_importer.py
  11. 17
      main_package/classes/structure.py
  12. 185
      main_package/classes/structure_estimator.py
  13. 10
      main_package/classes/trajectory.py
  14. 28
      main_package/tests/test_cache.py
  15. 96
      main_package/tests/test_json_importer.py
  16. 60
      main_package/tests/test_networkgraph.py
  17. 48
      main_package/tests/test_parameters_estimator.py
  18. 8
      main_package/tests/test_sample_path.py
  19. 32
      main_package/tests/test_setofcims.py
  20. 28
      main_package/tests/test_structure_estimator.py

@ -6,7 +6,6 @@ sys.path.append("./classes/")
import network_graph as ng
import sample_path as sp
import set_of_cims as sofc
import parameters_estimator as pe
import json_importer as ji
@ -14,20 +13,23 @@ import json_importer as ji
def main():
read_files = glob.glob(os.path.join('./data', "*.json")) #Take all json files in this dir
#import data
importer = ji.JsonImporter(read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
importer = ji.JsonImporter(read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 1)
#Create a SamplePath Obj
s1 = sp.SamplePath(importer)
#Build The trajectries and the structural infos
s1.build_trajectories()
s1.build_structure()
print(s1.structure.edges)
print(s1.structure.nodes_values)
#From The Structure Object build the Graph
g = ng.NetworkGraph(s1.structure)
#Select a node you want to estimate the parameters
node = g.nodes[1]
#Init the graph specifically for THIS node
print("NOde", node)
#Init the _graph specifically for THIS node
g.fast_init(node)
#Use SamplePath and Grpah to create a ParametersEstimator Object
p1 = pe.ParametersEstimator(s1, g)
p1 = pe.ParametersEstimator(s1.trajectories, g)
#Init the peEst specifically for THIS node
p1.fast_init(node)
#Compute the parameters

@ -7,21 +7,21 @@ class AbstractImporter(ABC):
"""
Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
:file_path: the file path
:_file_path: the file path
:_concatenated_samples: the concatenation of all the processed trajectories
:df_structure: Dataframe containing the structure of the network (edges)
:df_variables: Dataframe containing the nodes cardinalities
:df_concatenated_samples: the concatenation and processing of all the trajectories present
:_df_structure: Dataframe containing the structure of the network (edges)
:_df_variables: Dataframe containing the nodes cardinalities
:_df_concatenated_samples: the concatenation and processing of all the trajectories present
in the list df_samples list
:sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples
:_sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples
"""
def __init__(self, file_path: str):
"""
Parameters:
:file_path: the path to the file containing the data
:_file_path: the path to the file containing the data
"""
self.file_path = file_path
self._file_path = file_path
self._df_variables = None
self._df_structure = None
self._concatenated_samples = None
@ -71,7 +71,6 @@ class AbstractImporter(ABC):
pre: the Dataframe sample_frame has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values]
"""
#sample_frame[time_header_label] = sample_frame[time_header_label].diff().shift(-1)
sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1)
shifted_cols = sample_frame[columns_header].shift(-1).fillna(0).astype('int32')
shifted_cols.columns = shifted_cols_header
@ -81,7 +80,7 @@ class AbstractImporter(ABC):
def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List):
"""
Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list df_samples_list.
Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list _df_samples_list.
Concatenates the result in the dataframe concatanated_samples
Parameters:
time_header_label: the label of the time column
@ -103,9 +102,7 @@ class AbstractImporter(ABC):
complete_header = self._sorter[:]
complete_header.insert(0,'Time')
complete_header.extend(shifted_cols_header)
#print("Complete Header", complete_header)
self._concatenated_samples = self._concatenated_samples[complete_header]
#print("Concat Samples",self._concatenated_samples)
def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List:
"""
@ -143,3 +140,7 @@ class AbstractImporter(ABC):
@property
def sorter(self) -> typing.List:
return self._sorter
@property
def file_path(self) -> str:
return self._file_path

@ -7,28 +7,28 @@ class Cache:
"""
This class has the role of a cache for SetOfCIMS of a test node that have been already computed during the ctpc algorithm.
:list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related
:actual_cache: a list of setOfCims objects
:_list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related
:_actual_cache: a list of setOfCims objects
"""
def __init__(self):
self.list_of_sets_of_parents = []
self.actual_cache = []
self._list_of_sets_of_parents = []
self._actual_cache = []
def find(self, parents_comb: typing.Set): #typing.Union[typing.Set, str]
def find(self, parents_comb: typing.Set) -> sofc.SetOfCims:
"""
Tries to find in cache given the symbolic parents combination parents_comb the SetOfCims related to that parents_comb.
Parameters:
parents_comb: the parents related to that SetOfCims
Returns:
A SetOfCims object if the parents_comb index is found in list_of_sets_of_parents.
A SetOfCims object if the parents_comb index is found in _list_of_sets_of_parents.
None otherwise.
"""
try:
#print("Cache State:", self.list_of_sets_of_indxs)
#print("Look For:", parents_comb)
result = self.actual_cache[self.list_of_sets_of_parents.index(parents_comb)]
result = self._actual_cache[self._list_of_sets_of_parents.index(parents_comb)]
print("CACHE HIT!!!!", parents_comb)
return result
except ValueError:
@ -36,7 +36,7 @@ class Cache:
def put(self, parents_comb: typing.Union[typing.Set, str], socim: sofc.SetOfCims):
"""
Place in cache the SetOfCims object, and the related sybolyc index parents_comb in list_of_sets_of_parents
Place in cache the SetOfCims object, and the related sybolyc index parents_comb in _list_of_sets_of_parents
Parameters:
parents_comb: the symbolic set index
@ -45,9 +45,9 @@ class Cache:
Returns:
void
"""
#print("Putting in cache:", parents_comb)
self.list_of_sets_of_parents.append(parents_comb)
self.actual_cache.append(socim)
#print("Putting in _cache:", parents_comb)
self._list_of_sets_of_parents.append(parents_comb)
self._actual_cache.append(socim)
def clear(self):
"""
@ -58,5 +58,5 @@ class Cache:
Returns:
void
"""
del self.list_of_sets_of_parents[:]
del self.actual_cache[:]
del self._list_of_sets_of_parents[:]
del self._actual_cache[:]

@ -11,6 +11,11 @@ class ConditionalIntensityMatrix:
:_cim: the actual cim of the node
"""
def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array):
"""
Parameters:
:_state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix
"""
self._state_residence_times = state_residence_times
self._state_transition_matrix = state_transition_matrix
self._cim = self.state_transition_matrix.astype(np.float64)
@ -28,15 +33,15 @@ class ConditionalIntensityMatrix:
self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T
@property
def state_residence_times(self):
def state_residence_times(self) -> np.ndarray:
return self._state_residence_times
@property
def state_transition_matrix(self):
def state_transition_matrix(self) -> np.ndarray:
return self._state_transition_matrix
@property
def cim(self):
def cim(self) -> np.ndarray:
return self._cim
def __repr__(self):

@ -16,31 +16,33 @@ class JsonImporter(ai.AbstractImporter):
|_ samples
|_ variabels
:file_path: the path of the file that contains tha data to be imported
:samples_label: the reference key for the samples in the trajectories
:structure_label: the reference key for the structure of the network data
:variables_label: the reference key for the cardinalites of the nodes data
:time_key: the key used to identify the timestamps in each trajectory
:variables_key: the key used to identify the names of the variables in the net
:df_samples_list: a Dataframe list in which every df contains a trajectory
:_samples_label: the reference key for the samples in the trajectories
:_structure_label: the reference key for the structure of the network data
:_variables_label: the reference key for the cardinalites of the nodes data
:_time_key: the key used to identify the timestamps in each trajectory
:_variables_key: the key used to identify the names of the variables in the net
:_df_samples_list: a Dataframe list in which every df contains a trajectory
"""
def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str,
variables_key: str):
variables_key: str, array_indx: int):
"""
Parameters:
file_path: the path of the file that contains tha data to be imported
:samples_label: the reference key for the samples in the trajectories
:structure_label: the reference key for the structure of the network data
:variables_label: the reference key for the cardinalites of the nodes data
:time_key: the key used to identify the timestamps in each trajectory
:variables_key: the key used to identify the names of the variables in the net
"""
self.samples_label = samples_label
self.structure_label = structure_label
self.variables_label = variables_label
self.time_key = time_key
self.variables_key = variables_key
self.df_samples_list = None
:file_path: the path of the file that contains tha data to be imported
:_samples_label: the reference key for the samples in the trajectories
:_structure_label: the reference key for the structure of the network data
:_variables_label: the reference key for the cardinalites of the nodes data
:_time_key: the key used to identify the timestamps in each trajectory
:_variables_key: the key used to identify the names of the variables in the net
:_array_indx: the index of the outer json array from which import all the data
"""
self._samples_label = samples_label
self._structure_label = structure_label
self._variables_label = variables_label
self._time_key = time_key
self._variables_key = variables_key
self._df_samples_list = None
self._array_indx = array_indx
super(JsonImporter, self).__init__(file_path)
def import_data(self):
@ -52,14 +54,14 @@ class JsonImporter(ai.AbstractImporter):
_void
"""
raw_data = self.read_json_file()
self.df_samples_list = self.import_trajectories(raw_data)
self._sorter = self.build_sorter(self.df_samples_list[0])
self.compute_row_delta_in_all_samples_frames(self.df_samples_list)
self._df_samples_list = self.import_trajectories(raw_data)
self._sorter = self.build_sorter(self._df_samples_list[0])
self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
self.clear_data_frame_list()
self._df_structure = self.import_structure(raw_data)
self._df_variables = self.import_variables(raw_data, self._sorter)
self._df_variables = self.import_variables(raw_data)
def import_trajectories(self, raw_data: typing.List):
def import_trajectories(self, raw_data: typing.List) -> typing.List:
"""
Imports the trajectories in the list of dicts raw_data.
Parameters:
@ -67,22 +69,22 @@ class JsonImporter(ai.AbstractImporter):
Returns:
:List of dataframes containing all the trajectories
"""
return self.normalize_trajectories(raw_data, 0, self.samples_label)
return self.normalize_trajectories(raw_data, self._array_indx, self._samples_label)
def import_structure(self, raw_data: typing.List) -> pd.DataFrame:
"""
Imports in a dataframe the data in the list raw_data at the key structure_label
Imports in a dataframe the data in the list raw_data at the key _structure_label
Parameters:
:raw_data: the data
Returns:
:Daframe containg the starting node a ending node of every arc of the network
"""
return self.one_level_normalizing(raw_data, 0, self.structure_label)
return self.one_level_normalizing(raw_data, self._array_indx, self._structure_label)
def import_variables(self, raw_data: typing.List, sorter: typing.List) -> pd.DataFrame:
def import_variables(self, raw_data: typing.List) -> pd.DataFrame:
"""
Imports the data in raw_data at the key variables_label.
Imports the data in raw_data at the key _variables_label.
Sorts the row of the dataframe df_variables using the list sorter.
Parameters:
@ -91,16 +93,7 @@ class JsonImporter(ai.AbstractImporter):
Returns:
:Datframe containg the variables simbolic labels and their cardinalities
"""
return self.one_level_normalizing(raw_data, 0, self.variables_label)
#TODO Usando come Pre-requisito l'ordinamento del frame _df_variables uguale a quello presente in
#TODO self _sorter questo codice risulta inutile
"""self._df_variables[self.variables_key] = self._df_variables[self.variables_key].astype("category")
self._df_variables[self.variables_key] = self._df_variables[self.variables_key].cat.set_categories(sorter)
self._df_variables = self._df_variables.sort_values([self.variables_key])
self._df_variables.reset_index(inplace=True)
self._df_variables.drop('index', axis=1, inplace=True)
#print("Var Frame", self._df_variables)
"""
return self.one_level_normalizing(raw_data, self._array_indx, self._variables_label)
def read_json_file(self) -> typing.List:
"""
@ -112,7 +105,7 @@ class JsonImporter(ai.AbstractImporter):
:data: the contents of the json file
"""
with open(self.file_path) as f:
with open(self._file_path) as f:
data = json.load(f)
return data
@ -130,7 +123,7 @@ class JsonImporter(ai.AbstractImporter):
"""
return pd.DataFrame(raw_data[indx][key])
def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str):
def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str) -> typing.List:
"""
Extracts the traj in raw_data at the index index at the key trajectories key.
@ -145,28 +138,25 @@ class JsonImporter(ai.AbstractImporter):
smps = raw_data[indx][trajectories_key]
df_samples_list = [dataframe(sample) for sample in smps]
return df_samples_list
#columns_header = list(self.df_samples_list[0].columns.values)
#columns_header.remove(self.time_key)
#self._sorter = columns_header
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
"""
Implements the abstract method build_sorter for this dataset
"""
columns_header = list(sample_frame.columns.values)
columns_header.remove(self.time_key)
columns_header.remove(self._time_key)
return columns_header
def clear_data_frame_list(self):
"""
Removes all values present in the dataframes in the list df_samples_list
Removes all values present in the dataframes in the list _df_samples_list
Parameters:
:void
Returns:
:void
"""
for indx in range(len(self.df_samples_list)):
self.df_samples_list[indx] = self.df_samples_list[indx].iloc[0:0]
for indx in range(len(self._df_samples_list)):
self._df_samples_list[indx] = self._df_samples_list[indx].iloc[0:0]
def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict:
"""

@ -1,37 +1,42 @@
import typing
import structure as st
import networkx as nx
import numpy as np
class NetworkGraph:
"""
Abstracts the infos contained in the Structure class in the form of a directed graph.
Abstracts the infos contained in the Structure class in the form of a directed _graph.
Has the task of creating all the necessary filtering structures for parameters estimation
:graph_struct: the Structure object from which infos about the net will be extracted
:graph: directed graph
:_graph_struct: the Structure object from which infos about the net will be extracted
:_graph: directed _graph
:nodes_labels: the symbolic names of the variables
:nodes_indexes: the indexes of the nodes
:nodes_values: the cardinalites of the nodes
:aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every
:_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every
node in the net
:_fancy_indexing: the indexes of every parent of every node in the net
:_time_scalar_indexing_structure: the indexing structure for state res time estimation
:_transition_scalar_indexing_structure: the indexing structure for transition computation
:_time_filtering: the columns filtering structure used in the computation of the state res times
:_transition_filtering: the columns filtering structure used in the computation of the transition from one state to another
:_transition_filtering: the columns filtering structure used in the computation of the transition
from one state to another
:self._p_combs_structure: all the possible parents states combination for every node in the net
"""
def __init__(self, graph_struct):
self.graph_struct = graph_struct
self.graph = nx.DiGraph()
self._nodes_indexes = self.graph_struct.nodes_indexes
self._nodes_labels = self.graph_struct.nodes_labels
self._nodes_values = self.graph_struct.nodes_values
self.aggregated_info_about_nodes_parents = None
def __init__(self, graph_struct: st.Structure):
"""
Parameters:
:graph_struct:the Structure object from which infos about the net will be extracted
"""
self._graph_struct = graph_struct
self._graph = nx.DiGraph()
self._nodes_indexes = self._graph_struct.nodes_indexes
self._nodes_labels = self._graph_struct.nodes_labels
self._nodes_values = self._graph_struct.nodes_values
self._aggregated_info_about_nodes_parents = None
self._fancy_indexing = None
self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None
@ -39,16 +44,6 @@ class NetworkGraph:
self._transition_filtering = None
self._p_combs_structure = None
def init_graph(self):
self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.edges)
self.aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures()
self.build_time_columns_filtering_structure()
self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
def fast_init(self, node_id: str):
"""
Initializes all the necessary structures for parameters estimation of the node identified by the label node_id
@ -58,11 +53,11 @@ class NetworkGraph:
void
"""
self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.edges)
self.aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
self._fancy_indexing = self.aggregated_info_about_nodes_parents[1]
self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
self._fancy_indexing = self._aggregated_info_about_nodes_parents[1]
p_indxs = self._fancy_indexing
p_vals = self.aggregated_info_about_nodes_parents[2]
p_vals = self._aggregated_info_about_nodes_parents[2]
self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id,
p_vals)
self._transition_scalar_indexing_structure = self.build_transition_scalar_indexing_structure_for_a_node(node_id,
@ -74,33 +69,33 @@ class NetworkGraph:
def add_nodes(self, list_of_nodes: typing.List):
"""
Adds the nodes to the graph contained in the list of nodes list_of_nodes.
Adds the nodes to the _graph contained in the list of nodes list_of_nodes.
Sets all the properties that identify a nodes (index, positional index, cardinality)
Parameters:
list_of_nodes: the nodes to add to graph
list_of_nodes: the nodes to add to _graph
Returns:
void
"""
nodes_indxs = self._nodes_indexes
nodes_vals = self.graph_struct.nodes_values
nodes_vals = self._graph_struct.nodes_values
pos = 0
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self.graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1
def add_edges(self, list_of_edges: typing.List):
"""
Add the edges to the graph contained in the list list_of_edges.
Add the edges to the _graph contained in the list list_of_edges.
Parameters:
list_of_edges
Returns:
void
"""
self.graph.add_edges_from(list_of_edges)
self._graph.add_edges_from(list_of_edges)
def get_ordered_by_indx_set_of_parents(self, node: str):
def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple:
"""
Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
(parents_labels, parents_indexes, parents_cardinalities).
@ -121,22 +116,6 @@ class NetworkGraph:
p_values = [self.get_states_number(node) for node in sorted_parents]
return (sorted_parents, p_indxes, p_values)
def get_ord_set_of_par_of_all_nodes(self):
get_ordered_by_indx_set_of_parents = self.get_ordered_by_indx_set_of_parents
result = [get_ordered_by_indx_set_of_parents(node) for node in self._nodes_labels]
return result
def get_ordered_by_indx_parents_values_for_all_nodes(self):
pars_values = [i[2] for i in self.aggregated_info_about_nodes_parents]
return pars_values
def build_fancy_indexing_structure(self, start_indx):
if start_indx > 0:
pass
else:
fancy_indx = [i[1] for i in self.aggregated_info_about_nodes_parents]
return fancy_indx
def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray:
"""
Builds an indexing structure for the computation of state residence times values.
@ -153,8 +132,8 @@ class NetworkGraph:
T_vector = T_vector.cumprod().astype(np.int)
return T_vector
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray:
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) \
-> np.ndarray:
"""
Builds an indexing structure for the computation of state transitions values.
@ -194,7 +173,7 @@ class NetworkGraph:
Returns:
a numpy array
"""
nodes_number = self.graph_struct.total_variables_number
nodes_number = self._graph_struct.total_variables_number
return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray:
@ -219,81 +198,108 @@ class NetworkGraph:
parents_comb = np.array([[]], dtype=np.int)
return parents_comb
def build_time_columns_filtering_structure(self):
nodes_indxs = self._nodes_indexes
self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)]
def build_transition_columns_filtering_structure(self):
nodes_number = self.graph_struct.total_variables_number
nodes_indxs = self._nodes_indexes
self._transition_filtering = [np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
for node_indx, p_indxs in zip(nodes_indxs,
self._fancy_indexing)]
def build_scalar_indexing_structures(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
build_transition_scalar_indexing_structure_for_a_node = self.build_transition_scalar_indexing_structure_for_a_node
build_time_scalar_indexing_structure_for_a_node = self.build_time_scalar_indexing_structure_for_a_node
aggr = [(build_transition_scalar_indexing_structure_for_a_node(node_id, p_vals),
build_time_scalar_indexing_structure_for_a_node(node_id, p_vals))
for node_id, p_vals in
zip(self._nodes_labels,
parents_values_for_all_nodes)]
self._transition_scalar_indexing_structure = [i[0] for i in aggr]
self._time_scalar_indexing_structure = [i[1] for i in aggr]
def build_p_combs_structure(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs_struct = [self.build_p_comb_structure_for_a_node(p_vals) for p_vals in parents_values_for_all_nodes]
return p_combs_struct
def get_parents_by_id(self, node_id):
return list(self.graph.predecessors(node_id))
return list(self._graph.predecessors(node_id))
def get_states_number(self, node_id):
return self.graph.nodes[node_id]['val']
return self._graph.nodes[node_id]['val']
def get_node_indx(self, node_id):
return nx.get_node_attributes(self.graph, 'indx')[node_id]
return nx.get_node_attributes(self._graph, 'indx')[node_id]
def get_positional_node_indx(self, node_id):
return self.graph.nodes[node_id]['pos_indx']
return self._graph.nodes[node_id]['pos_indx']
@property
def nodes(self):
def nodes(self) -> typing.List:
return self._nodes_labels
@property
def edges(self):
return list(self.graph.edges)
def edges(self) -> typing.List:
return list(self._graph.edges)
@property
def nodes_indexes(self):
def nodes_indexes(self) -> np.ndarray:
return self._nodes_indexes
@property
def nodes_values(self):
def nodes_values(self) -> np.ndarray:
return self._nodes_values
@property
def time_scalar_indexing_strucure(self):
def time_scalar_indexing_strucure(self) -> np.ndarray:
return self._time_scalar_indexing_structure
@property
def time_filtering(self):
def time_filtering(self) -> np.ndarray:
return self._time_filtering
@property
def transition_scalar_indexing_structure(self):
def transition_scalar_indexing_structure(self) -> np.ndarray:
return self._transition_scalar_indexing_structure
@property
def transition_filtering(self):
def transition_filtering(self) -> np.ndarray:
return self._transition_filtering
@property
def p_combs(self):
def p_combs(self) -> np.ndarray:
return self._p_combs_structure
"""##############These Methods are actually unused but could become useful in the near future################"""
def init_graph(self):
self.add_nodes(self._nodes_labels)
self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures()
self.build_time_columns_filtering_structure()
self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
def build_time_columns_filtering_structure(self):
nodes_indxs = self._nodes_indexes
self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)]
def build_transition_columns_filtering_structure(self):
nodes_number = self._graph_struct.total_variables_number
nodes_indxs = self._nodes_indexes
self._transition_filtering = [np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
for node_indx, p_indxs in zip(nodes_indxs,
self._fancy_indexing)]
def build_scalar_indexing_structures(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
build_transition_scalar_indexing_structure_for_a_node = \
self.build_transition_scalar_indexing_structure_for_a_node
build_time_scalar_indexing_structure_for_a_node = self.build_time_scalar_indexing_structure_for_a_node
aggr = [(build_transition_scalar_indexing_structure_for_a_node(node_id, p_vals),
build_time_scalar_indexing_structure_for_a_node(node_id, p_vals))
for node_id, p_vals in
zip(self._nodes_labels,
parents_values_for_all_nodes)]
self._transition_scalar_indexing_structure = [i[0] for i in aggr]
self._time_scalar_indexing_structure = [i[1] for i in aggr]
def build_p_combs_structure(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs_struct = [self.build_p_comb_structure_for_a_node(p_vals) for p_vals in parents_values_for_all_nodes]
return p_combs_struct
def get_ord_set_of_par_of_all_nodes(self):
get_ordered_by_indx_set_of_parents = self.get_ordered_by_indx_set_of_parents
result = [get_ordered_by_indx_set_of_parents(node) for node in self._nodes_labels]
return result
def get_ordered_by_indx_parents_values_for_all_nodes(self):
pars_values = [i[2] for i in self._aggregated_info_about_nodes_parents]
return pars_values
def build_fancy_indexing_structure(self, start_indx):
if start_indx > 0:
pass
else:
fancy_indx = [i[1] for i in self._aggregated_info_about_nodes_parents]
return fancy_indx

@ -2,7 +2,7 @@
import numpy as np
import network_graph as ng
import sample_path as sp
import trajectory as tr
import set_of_cims as sofc
import sets_of_cims_container as acims
@ -10,18 +10,24 @@ import sets_of_cims_container as acims
class ParametersEstimator:
"""
Has the task of computing the cims of particular node given the trajectories in samplepath and the net structure
in the graph net_graph
in the graph _net_graph
:sample_path: the container of the trajectories
:net_graph: the net structure
:single_srt_of_cims: the set of cims object that will hold the cims of the node
:trajectories: the trajectories
:_net_graph: the net structure
:_single_set_of_cims: the set of cims object that will hold the cims of the node
"""
def __init__(self, sample_path: sp.SamplePath, net_graph: ng.NetworkGraph):
self.sample_path = sample_path
self.net_graph = net_graph
self.sets_of_cims_struct = None
self.single_set_of_cims = None
def __init__(self, trajectories: tr.Trajectory, net_graph: ng.NetworkGraph):
"""
Parameters:
:trajectories: the trajectories
:_net_graph: the net structure
"""
#self.sample_path = sample_path
self._trajectories = trajectories
self._net_graph = net_graph
#self.sets_of_cims_struct = None
self._single_set_of_cims = None
def fast_init(self, node_id: str):
"""
@ -32,9 +38,9 @@ class ParametersEstimator:
Returns:
void
"""
p_vals = self.net_graph.aggregated_info_about_nodes_parents[2]
node_states_number = self.net_graph.get_states_number(node_id)
self.single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self.net_graph.p_combs)
p_vals = self._net_graph._aggregated_info_about_nodes_parents[2]
node_states_number = self._net_graph.get_states_number(node_id)
self._single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs)
def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims:
"""
@ -45,22 +51,22 @@ class ParametersEstimator:
Returns:
A setOfCims object filled with the computed CIMS
"""
node_indx = self.net_graph.get_node_indx(node_id)
state_res_times = self.single_set_of_cims.state_residence_times
transition_matrices = self.single_set_of_cims.transition_matrices
trajectory = self.sample_path.trajectories.trajectory
self.compute_state_res_time_for_node(node_indx, self.sample_path.trajectories.times,
trajectory,
self.net_graph.time_filtering,
self.net_graph.time_scalar_indexing_strucure,
node_indx = self._net_graph.get_node_indx(node_id)
state_res_times = self._single_set_of_cims._state_residence_times
transition_matrices = self._single_set_of_cims._transition_matrices
#trajectory = self.sample_path.trajectories.trajectory
self.compute_state_res_time_for_node(node_indx, self._trajectories.times,
self._trajectories.trajectory,
self._net_graph.time_filtering,
self._net_graph.time_scalar_indexing_strucure,
state_res_times)
self.compute_state_transitions_for_a_node(node_indx,
self.sample_path.trajectories.complete_trajectory,
self.net_graph.transition_filtering,
self.net_graph.transition_scalar_indexing_structure,
self._trajectories.complete_trajectory,
self._net_graph.transition_filtering,
self._net_graph.transition_scalar_indexing_structure,
transition_matrices)
self.single_set_of_cims.build_cims(state_res_times, transition_matrices)
return self.single_set_of_cims
self._single_set_of_cims.build_cims(state_res_times, transition_matrices)
return self._single_set_of_cims
def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray,
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray, T: np.ndarray):
@ -99,38 +105,44 @@ class ParametersEstimator:
diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])],
dtype=np.int64)
trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0]
#print("Trajectory", trajectory)
#print("Step 1", trajectory[:, int(trajectory.shape[1] / 2) + node_indx])
#print("Step 2", trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0)
#print("TrTemp", trj_tmp)
#print("Cols Filter", cols_filter)
#print("Filtered Tr Temp", trj_tmp[:, cols_filter])
#print("Actual Indexing", scalar_indexing / scalar_indexing[0])
#print("PreBins",trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0] )
#print("Bins", np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1))
#print("After BinCount", np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int)))
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int),
minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
M_raveled = M.ravel()
M_raveled[diag_indices] = 0
M_raveled[diag_indices] = np.sum(M, axis=2).ravel()
"""##############These Methods are actually unused but could become useful in the near future################"""
def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self.net_graph.nodes,
self.net_graph.nodes_values,
self.net_graph.get_ordered_by_indx_parents_values_for_all_nodes(),
self.net_graph.p_combs)
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self._net_graph.nodes,
self._net_graph.nodes_values,
self._net_graph.
get_ordered_by_indx_parents_values_for_all_nodes(),
self._net_graph.p_combs)
def compute_parameters(self):
#print(self.net_graph.get_nodes())
#print(self.amalgamated_cims_struct.sets_of_cims)
#enumerate(zip(self.net_graph.get_nodes(), self.amalgamated_cims_struct.sets_of_cims))
for indx, aggr in enumerate(zip(self.net_graph.nodes, self.sets_of_cims_struct.sets_of_cims)):
#print(self.net_graph.time_filtering[indx])
#print(self.net_graph.time_scalar_indexing_strucure[indx])
self.compute_state_res_time_for_node(self.net_graph.get_node_indx(aggr[0]), self.sample_path.trajectories.times,
for indx, aggr in enumerate(zip(self._net_graph.nodes, self.sets_of_cims_struct.sets_of_cims)):
self.compute_state_res_time_for_node(self._net_graph.get_node_indx(aggr[0]), self.sample_path.trajectories.times,
self.sample_path.trajectories.trajectory,
self.net_graph.time_filtering[indx],
self.net_graph.time_scalar_indexing_strucure[indx],
aggr[1].state_residence_times)
#print(self.net_graph.transition_filtering[indx])
#print(self.net_graph.transition_scalar_indexing_structure[indx])
self.compute_state_transitions_for_a_node(self.net_graph.get_node_indx(aggr[0]),
self._net_graph.time_filtering[indx],
self._net_graph.time_scalar_indexing_strucure[indx],
aggr[1]._state_residence_times)
self.compute_state_transitions_for_a_node(self._net_graph.get_node_indx(aggr[0]),
self.sample_path.trajectories.complete_trajectory,
self.net_graph.transition_filtering[indx],
self.net_graph.transition_scalar_indexing_structure[indx],
aggr[1].transition_matrices)
aggr[1].build_cims(aggr[1].state_residence_times, aggr[1].transition_matrices)
self._net_graph.transition_filtering[indx],
self._net_graph.transition_scalar_indexing_structure[indx],
aggr[1]._transition_matrices)
aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices)

@ -10,10 +10,10 @@ class SamplePath:
cardinalites.
Has the task of creating the objects that will contain the mentioned data.
:importer: the Importer objects that will import ad process data
:_importer: the Importer objects that will import ad process data
:trajectories: the Trajectory object that will contain all the concatenated trajectories
:structure: the Structure Object that will contain all the structurral infos about the net
:total_variables_count: the number of variables in the net
:_total_variables_count: the number of variables in the net
"""
def __init__(self, importer: imp.AbstractImporter):
@ -21,11 +21,11 @@ class SamplePath:
Parameters:
:importer: the Importer objects that will import ad process data
"""
self.importer = importer
self._importer = importer
self._trajectories = None
self._structure = None
self.total_variables_count = None
self.importer.import_data()
self._total_variables_count = None
self._importer.import_data()
def build_trajectories(self):
"""
@ -37,12 +37,12 @@ class SamplePath:
Returns:
:void
"""
#self.importer.import_data()
#self._importer.import_data()
self._trajectories = \
tr.Trajectory(self.importer.build_list_of_samples_array(self.importer.concatenated_samples),
len(self.importer.sorter) + 1)
tr.Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples),
len(self._importer.sorter) + 1)
#self.trajectories.append(trajectory)
self.importer.clear_concatenated_frame()
self._importer.clear_concatenated_frame()
def build_structure(self):
"""
@ -52,19 +52,16 @@ class SamplePath:
Returns:
:void
"""
if self.importer.sorter != self.importer.variables.iloc[:, 0].to_list():
if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list():
raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!")
self.total_variables_count = len(self.importer.sorter)
#labels = self.importer.variables[self.importer.variables_key].to_list()
#print("SAMPLE PATH LABELS",labels)
#print(self.importer.variables)
labels = self.importer.variables.iloc[:, 0].to_list()
indxs = self.importer.variables.index.to_numpy()
vals = self.importer.variables.iloc[:, 1].to_numpy()
edges = list(self.importer.structure.to_records(index=False))
self._total_variables_count = len(self._importer.sorter)
labels = self._importer.variables.iloc[:, 0].to_list()
indxs = self._importer.variables.index.to_numpy()
vals = self._importer.variables.iloc[:, 1].to_numpy()
edges = list(self._importer.structure.to_records(index=False))
self._structure = st.Structure(labels, indxs, vals, edges,
self.total_variables_count)
self._total_variables_count)
@property
def trajectories(self) -> tr.Trajectory:
@ -74,8 +71,9 @@ class SamplePath:
def structure(self) -> st.Structure:
return self._structure
@property
def total_variables_count(self):
return self.total_variables_count
return self._total_variables_count

@ -7,26 +7,33 @@ import conditional_intensity_matrix as cim
class SetOfCims:
"""
Aggregates all the CIMS of the node identified by the label node_id.
:node_id: the node label
:parents_states_number: the cardinalities of the parents
:node_states_number: the caridinality of the node
:p_combs: the relative p_comb structure
Aggregates all the CIMS of the node identified by the label _node_id.
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
:state_residence_time: matrix containing all the state residence time vectors for the node
:transition_matrices: matrix containing all the transition matrices for the node
:actaul_cims: the cims of the node
:_transition_matrices: matrix containing all the transition matrices for the node
:_actual_cims: the cims of the node
"""
def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray):
self.node_id = node_id
self.parents_states_number = parents_states_number
self.node_states_number = node_states_number
self.actual_cims = []
self.state_residence_times = None
self.transition_matrices = None
self.p_combs = p_combs
"""
Parameters:
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
"""
self._node_id = node_id
self._parents_states_number = parents_states_number
self._node_states_number = node_states_number
self._actual_cims = []
self._state_residence_times = None
self._transition_matrices = None
self._p_combs = p_combs
self.build_times_and_transitions_structures()
def build_times_and_transitions_structures(self):
@ -34,73 +41,64 @@ class SetOfCims:
Initializes at the correct dimensions the state residence times matrix and the state transition matrices
Parameters:
void
:void
Returns:
void
:void
"""
if not self.parents_states_number:
self.state_residence_times = np.zeros((1, self.node_states_number), dtype=np.float)
self.transition_matrices = np.zeros((1,self.node_states_number, self.node_states_number), dtype=np.int)
if not self._parents_states_number:
self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float)
self._transition_matrices = np.zeros((1, self._node_states_number, self._node_states_number), dtype=np.int)
else:
self.state_residence_times = \
np.zeros((np.prod(self.parents_states_number), self.node_states_number), dtype=np.float)
self.transition_matrices = np.zeros([np.prod(self.parents_states_number), self.node_states_number,
self.node_states_number], dtype=np.int)
self._state_residence_times = \
np.zeros((np.prod(self._parents_states_number), self._node_states_number), dtype=np.float)
self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number,
self._node_states_number], dtype=np.int)
def build_cims(self, state_res_times: typing.List, transition_matrices: typing.List):
def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray):
"""
Build the ConditionalIntensityMatrix object given the state residence times and transitions matrices.
Compute the cim coefficients.
Parameters:
state_res_times: the state residence times matrix
transition_matrices: the transition matrices
:state_res_times: the state residence times matrix
:_transition_matrices: the transition matrices
Returns:
void
:void
"""
for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices):
cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix)
cim_to_add.compute_cim_coefficients()
self.actual_cims.append(cim_to_add)
self.actual_cims = np.array(self.actual_cims)
self.transition_matrices = None
self.state_residence_times = None
self._actual_cims.append(cim_to_add)
self._actual_cims = np.array(self._actual_cims)
self._transition_matrices = None
self._state_residence_times = None
def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray:
"""
Filter the cims contained in the array actual_cims given the boolean mask mask_arr and the index comb.
Filter the cims contained in the array _actual_cims given the boolean mask mask_arr and the index comb.
Parameters:
mask_arr: the boolean mask
comb: the indexes of the selected cims
:mask_arr: the boolean mask
:comb: the indexes of the selected cims
Returns:
Array of ConditionalIntensityMatrix
:Array of ConditionalIntensityMatrix
"""
if mask_arr.size <= 1:
return self.actual_cims
return self._actual_cims
else:
flat_indxs = np.argwhere(np.all(self.p_combs[:, mask_arr] == comb, axis=1)).ravel()
return self.actual_cims[flat_indxs]
flat_indxs = np.argwhere(np.all(self._p_combs[:, mask_arr] == comb, axis=1)).ravel()
return self._actual_cims[flat_indxs]
@property
def get_cims(self):
return self.actual_cims
def actual_cims(self) -> np.ndarray:
return self._actual_cims
@property
def p_combs(self) -> np.ndarray:
return self._p_combs
def get_cims_number(self):
return len(self.actual_cims)
"""
def get_cim(self, index):
flat_index = self.indexes_converter(index)
return self.actual_cims[flat_index]
def indexes_converter(self, indexes):
assert len(indexes) == len(self.parents_states_number)
vector_index = 0
if not indexes:
return vector_index
else:
for indx, value in enumerate(indexes):
vector_index = vector_index*self.parents_states_number[indx] + indexes[indx]
return vector_index"""
return len(self._actual_cims)

@ -19,7 +19,7 @@ class CSVImporter(ai.AbstractImporter):
self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
def read_csv_file(self):
df = pd.read_csv(self.file_path)
df = pd.read_csv(self._file_path)
df.drop(df.columns[[0]], axis=1, inplace=True)
self._df_samples_list = [df]

@ -16,6 +16,15 @@ class Structure:
def __init__(self, nodes_label_list: ty.List, node_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
edges_list: ty.List, total_variables_number: int):
"""
Parameters:
:nodes_labels_list: the symbolic names of the variables
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
self._nodes_labels_list = nodes_label_list
self._nodes_indexes_arr = node_indexes_arr
self._nodes_vals_arr = nodes_vals_arr
@ -23,13 +32,11 @@ class Structure:
self._total_variables_number = total_variables_number
@property
def edges(self):
#records = self.structure_frame.to_records(index=False)
#edges_list = list(records)
def edges(self) -> ty.List:
return self._edges_list
@property
def nodes_labels(self):
def nodes_labels(self) -> ty.List:
return self._nodes_labels_list
@property
@ -41,7 +48,7 @@ class Structure:
return self._nodes_vals_arr
@property
def total_variables_number(self):
def total_variables_number(self) -> int:
return self._total_variables_number
def get_node_id(self, node_indx: int) -> str:

@ -21,28 +21,35 @@ class StructureEstimator:
"""
Has the task of estimating the network structure given the trajectories in samplepath.
:sample_path: the sample_path object containing the trajectories and the real structure
:exp_test_sign: the significance level for the exponential Hp test
:chi_test_alfa: the significance level for the chi Hp test
:nodes: the nodes labels
:nodes_vals: the nodes cardinalities
:nodes_indxs: the nodes indexes
:complete_graph: the complete directed graph built using the nodes labels in nodes
:cache: the cache object
:_sample_path: the _sample_path object containing the trajectories and the real structure
:_exp_test_sign: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test
:_nodes: the _nodes labels
:_nodes_vals: the nodes cardinalities
:_nodes_indxs: the nodes indexes
:_complete_graph: the complete directed graph built using the nodes labels in nodes
:_cache: the _cache object
"""
def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float):
self.sample_path = sample_path
self.nodes = np.array(self.sample_path.structure.nodes_labels)
self.nodes_vals = self.sample_path.structure.nodes_values
self.nodes_indxs = self.sample_path.structure.nodes_indexes
self.complete_graph = self.build_complete_graph(self.sample_path.structure.nodes_labels)
self.exp_test_sign = exp_test_alfa
self.chi_test_alfa = chi_test_alfa
self.cache = ch.Cache()
"""
Parameters:
:_sample_path: the _sample_path object containing the trajectories and the real structure
:_exp_test_sign: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test
def build_complete_graph(self, node_ids: typing.List):
"""
self._sample_path = sample_path
self._nodes = np.array(self._sample_path.structure.nodes_labels)
self._nodes_vals = self._sample_path.structure.nodes_values
self._nodes_indxs = self._sample_path.structure.nodes_indexes
self._complete_graph = self.build_complete_graph(self._sample_path.structure.nodes_labels)
self._exp_test_sign = exp_test_alfa
self._chi_test_alfa = chi_test_alfa
self._cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List) -> nx.DiGraph:
"""
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
@ -57,9 +64,9 @@ class StructureEstimator:
return complete_graph
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int):
tot_vars_count: int) -> bool:
"""
Permorms a complete independence test on the directed graphs G1 = test_child U parent_set
Performs a complete independence test on the directed graphs G1 = test_child U parent_set
G2 = G1 U test_parent (added as an additional parent of the test_child).
Generates all the necessary structures and datas to perform the tests.
@ -68,98 +75,61 @@ class StructureEstimator:
test_child: the node label of the child
parent_set: the common parent set
child_states_numb: the cardinality of the test_child
tot_vars_count_ the total number of variables in the net
tot_vars_count: the total number of variables in the net
Returns:
True iff test_child and test_parent are independent given the sep_set parent_set
False otherwise
"""
#print("Test Parent:", test_parent)
#print("Sep Set", parent_set)
p_set = parent_set[:]
complete_info = parent_set[:]
complete_info.append(test_child)
parents = np.array(parent_set)
parents = np.append(parents, test_parent)
#print("PARENTS", parents)
#parents.sort()
sorted_parents = self.nodes[np.isin(self.nodes, parents)]
#print("SORTED PARENTS", sorted_parents)
sorted_parents = self._nodes[np.isin(self._nodes, parents)]
cims_filter = sorted_parents != test_parent
#print("PARENTS NO FROM MASK", cims_filter)
#if not p_set:
#print("EMPTY PSET TRYING TO FIND", test_child)
#sofc1 = self.cache.find(test_child)
#else:
sofc1 = self.cache.find(set(p_set))
sofc1 = self._cache.find(set(p_set))
if not sofc1:
#print("CACHE MISSS SOFC1")
bool_mask1 = np.isin(self.nodes,complete_info)
#print("Bool mask 1", bool_mask1)
l1 = list(self.nodes[bool_mask1])
#print("L1", l1)
indxs1 = self.nodes_indxs[bool_mask1]
#print("INDXS 1", indxs1)
vals1 = self.nodes_vals[bool_mask1]
bool_mask1 = np.isin(self._nodes, complete_info)
l1 = list(self._nodes[bool_mask1])
indxs1 = self._nodes_indxs[bool_mask1]
vals1 = self._nodes_vals[bool_mask1]
eds1 = list(itertools.product(parent_set,test_child))
s1 = st.Structure(l1, indxs1, vals1, eds1, tot_vars_count)
g1 = ng.NetworkGraph(s1)
g1.fast_init(test_child)
p1 = pe.ParametersEstimator(self.sample_path, g1)
p1 = pe.ParametersEstimator(self._sample_path.trajectories, g1)
p1.fast_init(test_child)
sofc1 = p1.compute_parameters_for_node(test_child)
#if not p_set:
#self.cache.put(test_child, sofc1)
#else:
self.cache.put(set(p_set), sofc1)
self._cache.put(set(p_set), sofc1)
sofc2 = None
#p_set.append(test_parent)
p_set.insert(0, test_parent)
if p_set:
#print("FULL PSET TRYING TO FIND", p_set)
#p_set.append(test_parent)
#print("PSET ", p_set)
#set_p_set = set(p_set)
sofc2 = self.cache.find(set(p_set))
#if sofc2:
#print("Sofc2 in CACHE ", sofc2.actual_cims)
#print(self.cache.list_of_sets_of_indxs)
sofc2 = self._cache.find(set(p_set))
if not sofc2:
#print("Cache MISSS SOFC2")
complete_info.append(test_parent)
bool_mask2 = np.isin(self.nodes, complete_info)
#print("BOOL MASK 2",bool_mask2)
l2 = list(self.nodes[bool_mask2])
#print("L2", l2)
indxs2 = self.nodes_indxs[bool_mask2]
#print("INDXS 2", indxs2)
vals2 = self.nodes_vals[bool_mask2]
bool_mask2 = np.isin(self._nodes, complete_info)
l2 = list(self._nodes[bool_mask2])
indxs2 = self._nodes_indxs[bool_mask2]
vals2 = self._nodes_vals[bool_mask2]
eds2 = list(itertools.product(p_set, test_child))
s2 = st.Structure(l2, indxs2, vals2, eds2, tot_vars_count)
g2 = ng.NetworkGraph(s2)
g2.fast_init(test_child)
p2 = pe.ParametersEstimator(self.sample_path, g2)
p2 = pe.ParametersEstimator(self._sample_path.trajectories, g2)
p2.fast_init(test_child)
sofc2 = p2.compute_parameters_for_node(test_child)
self.cache.put(set(p_set), sofc2)
self._cache.put(set(p_set), sofc2)
for cim1, p_comb in zip(sofc1.actual_cims, sofc1.p_combs):
#print("GETTING THIS P COMB", p_comb)
#if len(parent_set) > 1:
cond_cims = sofc2.filter_cims_with_mask(cims_filter, p_comb)
#else:
#cond_cims = sofc2.actual_cims
#print("COnd Cims", cond_cims)
for cim2 in cond_cims:
#cim2 = sofc2.actual_cims[j]
#print(indx)
#print("Run Test", i, j)
if not self.independence_test(child_states_numb, cim1, cim2):
return False
return True
def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix,
cim2: condim.ConditionalIntensityMatrix):
cim2: condim.ConditionalIntensityMatrix) -> bool:
"""
Compute the actual independence test using two cims.
It is performed first the exponential test and if the null hypothesis is not rejected,
@ -181,38 +151,23 @@ class StructureEstimator:
C1 = cim1.cim
C2 = cim2.cim
F_stats = C2.diagonal() / C1.diagonal()
exp_alfa = self.exp_test_sign
exp_alfa = self._exp_test_sign
for val in range(0, child_states_numb):
if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \
F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]):
#print("CONDITIONALLY DEPENDENT EXP")
return False
#M1_no_diag = self.remove_diagonal_elements(cim1.state_transition_matrix)
#M2_no_diag = self.remove_diagonal_elements(cim2.state_transition_matrix)
M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1)
M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape(
M2.shape[0], -1)
chi_2_quantile = chi2_dist.ppf(1 - self.chi_test_alfa, child_states_numb - 1)
"""
Ks = np.sqrt(cim1.state_transition_matrix.diagonal() / cim2.state_transition_matrix.diagonal())
Ls = np.reciprocal(Ks)
chi_stats = np.sum((np.power((M2_no_diag.T * Ks).T - (M1_no_diag.T * Ls).T, 2) \
/ (M1_no_diag + M2_no_diag)), axis=1)"""
chi_2_quantile = chi2_dist.ppf(1 - self._chi_test_alfa, child_states_numb - 1)
Ks = np.sqrt(r1s / r2s)
Ls = np.sqrt(r2s / r1s)
for val in range(0, child_states_numb):
#K = math.sqrt(cim1.state_transition_matrix[val][val] / cim2.state_transition_matrix[val][val])
#L = 1 / K
Chi = np.sum(np.power(Ks[val] * M2_no_diag[val] - Ls[val] *M1_no_diag[val], 2) /
(M1_no_diag[val] + M2_no_diag[val]))
#print("Chi Stats", Chi)
#print("Chi Quantile", chi_2_quantile)
if Chi > chi_2_quantile:
#if np.any(chi_stats > chi_2_quantile):
#print("CONDITIONALLY DEPENDENT CHI")
return False
#print("Chi test", Chi)
return True
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int):
@ -221,54 +176,45 @@ class StructureEstimator:
Parameters:
var_id: the node label of the test child
tot_vars_count: the number of nodes in the net
tot_vars_count: the number of _nodes in the net
Returns:
void
"""
print("##################TESTING VAR################", var_id)
u = list(self.complete_graph.predecessors(var_id))
#tests_parents_numb = len(u)
#complete_frame = self.complete_graph_frame
#test_frame = complete_frame.loc[complete_frame['To'].isin([var_id])]
child_states_numb = self.sample_path.structure.get_states_number(var_id)
u = list(self._complete_graph.predecessors(var_id))
child_states_numb = self._sample_path.structure.get_states_number(var_id)
b = 0
while b < len(u):
#for parent_id in u:
parent_indx = 0
while parent_indx < len(u):
removed = False
#if not list(self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])):
#break
S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])
#print("U Set", u)
#print("S", S)
test_parent = u[parent_indx]
#print("Test Parent", test_parent)
for parents_set in S:
#print("Parent Set", parents_set)
#print("Test Parent", test_parent)
print("Parent Set", parents_set)
print("Test Parent", test_parent)
if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count):
#print("Removing EDGE:", test_parent, var_id)
self.complete_graph.remove_edge(test_parent, var_id)
self._complete_graph.remove_edge(test_parent, var_id)
u.remove(test_parent)
removed = True
break
#else:
#parent_indx += 1
if not removed:
parent_indx += 1
b += 1
self.cache.clear()
self._cache.clear()
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str):
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str) -> \
typing.Iterator:
"""
Creates a list containing all possible subsets of the list u of size size,
that do not contains a the node identified by parent_label.
Parameters:
u: the list of nodes
u: the list of _nodes
size: the size of the subsets
parent_label: the nodes to exclude in the subsets generation
parent_label: the _nodes to exclude in the subsets generation
Returns:
a Map Object containing a list of lists
@ -286,8 +232,8 @@ class StructureEstimator:
void
"""
ctpc_algo = self.one_iteration_of_CTPC_algorithm
total_vars_numb = self.sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self.nodes]
total_vars_numb = self._sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self._nodes]
def save_results(self):
"""
@ -298,17 +244,10 @@ class StructureEstimator:
Returns:
void
"""
res = json_graph.node_link_data(self.complete_graph)
name = self.sample_path.importer.file_path.rsplit('/',1)[-1]
#print(name)
name = 'results_' + name
res = json_graph.node_link_data(self._complete_graph)
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1]
name = 'results_' + name #TODO va aggiunto anche l'indice di array
with open(name, 'w') as f:
json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)

@ -15,9 +15,15 @@ class Trajectory:
"""
def __init__(self, list_of_columns, original_cols_number):
"""
Parameters:
:list_of_columns: the list containing the times array and values matrix
:original_cols_numb: total number of cols in the data
"""
if type(list_of_columns[0][0]) != np.float64:
raise TypeError('The first array in the list has to be Times')
self.original_cols_number = original_cols_number
self._original_cols_number = original_cols_number
self._actual_trajectory = np.array(list_of_columns[1:], dtype=np.int).T
self._times = np.array(list_of_columns[0], dtype=np.float)
@ -29,7 +35,7 @@ class Trajectory:
Returns:
a numpy matrix containing ONLY the original columns values, not the shifted ones
"""
return self._actual_trajectory[:, :self.original_cols_number]
return self._actual_trajectory[:, :self._original_cols_number]
@property
def complete_trajectory(self) -> np.ndarray:

@ -11,31 +11,31 @@ class TestCache(unittest.TestCase):
def test_init(self):
c1 = ch.Cache()
self.assertFalse(c1.list_of_sets_of_parents)
self.assertFalse(c1.actual_cache)
self.assertFalse(c1._list_of_sets_of_parents)
self.assertFalse(c1._actual_cache)
def test_put(self):
c1 = ch.Cache()
pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents))
self.assertEqual(sofc1, c1.actual_cache[0])
self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1._list_of_sets_of_parents))
self.assertEqual(sofc1, c1._actual_cache[0])
pset2 = {'X'}
sofc2 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset2, sofc2)
self.assertEqual(2, len(c1.actual_cache))
self.assertEqual(2, len(c1.list_of_sets_of_parents))
self.assertEqual(sofc2, c1.actual_cache[1])
self.assertEqual(2, len(c1._actual_cache))
self.assertEqual(2, len(c1._list_of_sets_of_parents))
self.assertEqual(sofc2, c1._actual_cache[1])
def test_find(self):
c1 = ch.Cache()
pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents))
self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1._list_of_sets_of_parents))
self.assertIsInstance(c1.find(pset1), soci.SetOfCims)
self.assertEqual(sofc1, c1.find(pset1))
self.assertIsInstance(c1.find({'Y', 'X'}), soci.SetOfCims)
@ -47,11 +47,11 @@ class TestCache(unittest.TestCase):
pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents))
self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1._list_of_sets_of_parents))
c1.clear()
self.assertFalse(c1.list_of_sets_of_parents)
self.assertFalse(c1.actual_cache)
self.assertFalse(c1._list_of_sets_of_parents)
self.assertFalse(c1._actual_cache)

@ -16,14 +16,14 @@ class TestJsonImporter(unittest.TestCase):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
def test_init(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
self.assertEqual(j1.samples_label, 'samples')
self.assertEqual(j1.structure_label, 'dyn.str')
self.assertEqual(j1.variables_label, 'variables')
self.assertEqual(j1.time_key, 'Time')
self.assertEqual(j1.variables_key, 'Name')
self.assertEqual(j1.file_path, self.read_files[0])
self.assertIsNone(j1.df_samples_list)
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
self.assertEqual(j1._samples_label, 'samples')
self.assertEqual(j1._structure_label, 'dyn.str')
self.assertEqual(j1._variables_label, 'variables')
self.assertEqual(j1._time_key, 'Time')
self.assertEqual(j1._variables_key, 'Name')
self.assertEqual(j1._file_path, self.read_files[0])
self.assertIsNone(j1._df_samples_list)
self.assertIsNone(j1.variables)
self.assertIsNone(j1.structure)
self.assertIsNone(j1.concatenated_samples)
@ -35,7 +35,7 @@ class TestJsonImporter(unittest.TestCase):
json.dump(data_set, f)
path = os.getcwd()
path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '')
j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
imported_data = j1.read_json_file()
self.assertTrue(self.ordered(data_set) == self.ordered(imported_data))
os.remove('data.json')
@ -43,32 +43,32 @@ class TestJsonImporter(unittest.TestCase):
def test_read_json_file_not_found(self):
path = os.getcwd()
path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '')
j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
self.assertRaises(FileNotFoundError, j1.read_json_file)
def test_normalize_trajectories(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
#print(raw_data)
df_samples_list = j1.normalize_trajectories(raw_data, 0, j1.samples_label)
self.assertEqual(len(df_samples_list), len(raw_data[0][j1.samples_label]))
#self.assertEqual(list(j1.df_samples_list[0].columns.values)[1:], j1.sorter)
df_samples_list = j1.normalize_trajectories(raw_data, 0, j1._samples_label)
self.assertEqual(len(df_samples_list), len(raw_data[0][j1._samples_label]))
#self.assertEqual(list(j1._df_samples_list[0].columns.values)[1:], j1.sorter)
def test_normalize_trajectories_wrong_indx(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
self.assertRaises(IndexError, j1.normalize_trajectories, raw_data, 474, j1.samples_label)
self.assertRaises(IndexError, j1.normalize_trajectories, raw_data, 474, j1._samples_label)
def test_normalize_trajectories_wrong_key(self):
j1 = ji.JsonImporter(self.read_files[0], 'sample', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'sample', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
self.assertRaises(KeyError, j1.normalize_trajectories, raw_data, 0, j1.samples_label)
self.assertRaises(KeyError, j1.normalize_trajectories, raw_data, 0, j1._samples_label)
def test_compute_row_delta_single_samples_frame(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data)
sample_frame = j1.df_samples_list[0]
j1._df_samples_list = j1.import_trajectories(raw_data)
sample_frame = j1._df_samples_list[0]
original_copy = sample_frame.copy()
columns_header = list(sample_frame.columns.values)
shifted_cols_header = [s + "S" for s in columns_header[1:]]
@ -87,27 +87,27 @@ class TestJsonImporter(unittest.TestCase):
np.array(original_copy.iloc[indx + 1][columns_header[1:]], dtype=int))
def test_compute_row_delta_in_all_frames(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1.df_samples_list)
self.assertEqual(list(j1.df_samples_list[0].columns.values),
list(j1.concatenated_samples.columns.values)[:len(list(j1.df_samples_list[0].columns.values))])
self.assertEqual(list(j1.concatenated_samples.columns.values)[0], j1.time_key)
j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1._df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1._df_samples_list)
self.assertEqual(list(j1._df_samples_list[0].columns.values),
list(j1.concatenated_samples.columns.values)[:len(list(j1._df_samples_list[0].columns.values))])
self.assertEqual(list(j1.concatenated_samples.columns.values)[0], j1._time_key)
def test_clear_data_frame_list(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1.df_samples_list)
j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1._df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1._df_samples_list)
j1.clear_data_frame_list()
for df in j1.df_samples_list:
for df in j1._df_samples_list:
self.assertTrue(df.empty)
def test_clear_concatenated_frame(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
j1.import_data()
j1.clear_concatenated_frame()
self.assertTrue(j1.concatenated_samples.empty)
@ -118,7 +118,7 @@ class TestJsonImporter(unittest.TestCase):
json.dump(data_set, f)
path = os.getcwd()
path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '')
j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
raw_data = j1.read_json_file()
frame = pd.DataFrame(raw_data)
col_list = j1.build_list_of_samples_array(frame)
@ -130,39 +130,33 @@ class TestJsonImporter(unittest.TestCase):
os.remove('data.json')
def test_import_variables(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
sorter = ['X', 'Y', 'Z']
raw_data = [{'variables':{"Name": ['X', 'Y', 'Z'], "value": [3, 3, 3]}}]
df_var = j1.import_variables(raw_data, sorter)
self.assertEqual(list(df_var[j1.variables_key]), sorter)
df_var = j1.import_variables(raw_data)
self.assertEqual(list(df_var[j1._variables_key]), sorter)
def test_import_structure(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = [{"dyn.str":[{"From":"X","To":"Z"},{"From":"Y","To":"Z"},{"From":"Z","To":"Y"}]}]
df_struct = j1.import_structure(raw_data)
#print(raw_data[0]['dyn.str'][0].items())
self.assertIsInstance(df_struct, pd.DataFrame)
def test_import_sampled_cims(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0])
j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1._df_samples_list[0])
cims = j1.import_sampled_cims(raw_data, 0, 'dyn.cims')
#j1.import_variables(raw_data, j1.sorter)
self.assertEqual(list(cims.keys()), j1.sorter)
def test_import_data(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 2)
j1.import_data()
#lp = LineProfiler()
#lp_wrapper = lp(j1.import_data)
#lp_wrapper()
#lp.print_stats()
#j1.import_data()
self.assertEqual(list(j1.variables[j1.variables_key]),
list(j1.concatenated_samples.columns.values[1:len(j1.variables[j1.variables_key]) + 1]))
self.assertEqual(list(j1.variables[j1._variables_key]),
list(j1.concatenated_samples.columns.values[1:len(j1.variables[j1._variables_key]) + 1]))
print(j1.variables)
print(j1.structure)
print(j1.concatenated_samples)

@ -12,20 +12,19 @@ import network_graph as ng
import json_importer as ji
class TestNetworkGraph(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()
cls.s1.build_structure()
def test_init(self):
g1 = ng.NetworkGraph(self.s1.structure)
self.assertEqual(self.s1.structure, g1.graph_struct)
self.assertIsInstance(g1.graph, nx.DiGraph)
self.assertEqual(self.s1.structure, g1._graph_struct)
self.assertIsInstance(g1._graph, nx.DiGraph)
self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes))
self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels)
self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values))
@ -59,7 +58,7 @@ class TestNetworkGraph(unittest.TestCase):
self.assertEqual(graph.get_node_indx(node), node_indx)
else:
for node, node_val in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[2]):
self.assertEqual(graph.graph_struct.get_states_number(node), node_val)
self.assertEqual(graph._graph_struct.get_states_number(node), node_val)
def test_get_ord_set_of_par_of_all_nodes(self):
g1 = ng.NetworkGraph(self.s1.structure)
@ -68,36 +67,7 @@ class TestNetworkGraph(unittest.TestCase):
sorted_list_of_par_lists = g1.get_ord_set_of_par_of_all_nodes()
for node, par_list in zip(g1.nodes, sorted_list_of_par_lists):
self.aux_aggregated_par_list_data(g1, node, par_list)
"""
def test_get_ordered_by_indx_parents_values_for_all_nodes(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
#print(g1.get_ordered_by_indx_parents_values_for_all_nodes())
parents_values_list = g1.get_ordered_by_indx_parents_values_for_all_nodes()
for pv1, aggr in zip(parents_values_list, g1.aggregated_info_about_nodes_parents):
self.assertEqual(pv1, aggr[2])
def test_get_states_number_of_all_nodes_sorted(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
nodes_cardinality_list = g1.get_states_number_of_all_nodes_sorted()
for val, node in zip(nodes_cardinality_list, g1.get_nodes_sorted_by_indx()):
self.assertEqual(val, g1.get_states_number(node))
def test_build_fancy_indexing_structure_no_offset(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
for par_indxs, aggr in zip(fancy_indx, g1.aggregated_info_about_nodes_parents):
self.assertEqual(par_indxs, aggr[1])
def test_build_fancy_indexing_structure_offset(self):
pass #TODO il codice di netgraph deve gestire questo caso"""
def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals):
time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals)
@ -130,30 +100,30 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_labels = [i[0] for i in g1.aggregated_info_about_nodes_parents]
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
for node_id, p_i ,p_l, p_v in zip(g1.graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
for node_id, p_i ,p_l, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node_id, p_i ,p_l, p_v)
def test_build_time_scalar_indexing_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
p_labels = [i[0] for i in g1.aggregated_info_about_nodes_parents]
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
#print(fancy_indx)
for node_id, p_indxs, p_labels, p_v in zip(g1.graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
for node_id, p_indxs, p_labels, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node_id, p_indxs, p_labels, p_v)
def test_build_time_columns_filtering_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_time_columns_filtering_structure()
t_filter = []
@ -170,13 +140,13 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_transition_columns_filtering_structure()
m_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing):
single_filter = []
single_filter.append(g1.get_node_indx(node_id) + g1.graph_struct.total_variables_number)
single_filter.append(g1.get_node_indx(node_id) + g1._graph_struct.total_variables_number)
single_filter.append(g1.get_node_indx(node_id))
single_filter.extend(p_indxs)
m_filter.append(np.array(single_filter))
@ -187,7 +157,7 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs = g1.build_p_combs_structure()
@ -219,7 +189,7 @@ class TestNetworkGraph(unittest.TestCase):
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
for node in g1.nodes:
self.assertListEqual(g1.get_parents_by_id(node), list(g1.graph.predecessors(node)))
self.assertListEqual(g1.get_parents_by_id(node), list(g1._graph.predecessors(node)))
def test_get_states_number(self):
g1 = ng.NetworkGraph(self.s1.structure)

@ -17,10 +17,14 @@ class TestParametersEstimatior(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.array_indx = 8
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name',
cls.array_indx)
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()
cls.s1.build_structure()
print(cls.s1.structure.edges)
print(cls.s1.structure.nodes_values)
cls.g1 = ng.NetworkGraph(cls.s1.structure)
cls.g1.init_graph()
@ -28,29 +32,26 @@ class TestParametersEstimatior(unittest.TestCase):
for node in self.g1.nodes:
g = ng.NetworkGraph(self.s1.structure)
g.fast_init(node)
p1 = pe.ParametersEstimator(self.s1, g)
self.assertEqual(p1.sample_path, self.s1)
self.assertEqual(p1.net_graph, g)
self.assertIsNone(p1.single_set_of_cims)
p1 = pe.ParametersEstimator(self.s1.trajectories, g)
self.assertEqual(p1._trajectories, self.s1.trajectories)
self.assertEqual(p1._net_graph, g)
self.assertIsNone(p1._single_set_of_cims)
p1.fast_init(node)
self.assertIsInstance(p1.single_set_of_cims, sofc.SetOfCims)
self.assertIsInstance(p1._single_set_of_cims, sofc.SetOfCims)
def test_compute_parameters_for_node(self):
for indx, node in enumerate(self.g1.nodes):
print(node)
g = ng.NetworkGraph(self.s1.structure)
g.fast_init(node)
p1 = pe.ParametersEstimator(self.s1, g)
p1 = pe.ParametersEstimator(self.s1.trajectories, g)
p1.fast_init(node)
sofc1 = p1.compute_parameters_for_node(node)
sampled_cims = self.aux_import_sampled_cims('dyn.cims')
sc = list(sampled_cims.values())
#print(sc[indx])
self.equality_of_cims_of_node(sc[indx], sofc1.actual_cims)
self.equality_of_cims_of_node(sc[indx], sofc1._actual_cims)
def equality_of_cims_of_node(self, sampled_cims, estimated_cims):
#print(sampled_cims)
#print(estimated_cims)
self.assertEqual(len(sampled_cims), len(estimated_cims))
for c1, c2 in zip(sampled_cims, estimated_cims):
self.cim_equality_test(c1, c2.cim)
@ -60,30 +61,9 @@ class TestParametersEstimatior(unittest.TestCase):
self.assertTrue(np.all(np.isclose(r1, r2, 1e-01, 1e-01) == True))
def aux_import_sampled_cims(self, cims_label):
i1 = ji.JsonImporter(self.read_files[0], '', '', '', '', '')
i1 = ji.JsonImporter(self.read_files[0], '', '', '', '', '', self.array_indx)
raw_data = i1.read_json_file()
return i1.import_sampled_cims(raw_data, 0, cims_label)
return i1.import_sampled_cims(raw_data, self.array_indx, cims_label)
"""
def test_init(self):
self.aux_test_init(self.s1, self.g1)
def test_init_sets_of_cims_container(self):
self.aux_test_init_sets_cims_container(self.s1, self.g1)
def aux_test_init(self, sample_p, graph):
pe1 = pe.ParametersEstimator(sample_p, graph)
self.assertEqual(sample_p, pe1.sample_path)
self.assertEqual(graph, pe1.net_graph)
self.assertIsNone(pe1.sets_of_cims_struct)
def aux_test_init_sets_cims_container(self, sample_p, graph):
pe1 = pe.ParametersEstimator(sample_p, graph)
pe1.init_sets_cims_container()
self.assertIsInstance(pe1.sets_of_cims_struct, scc.SetsOfCimsContainer)
def test_compute_parameters(self):
self.aux_test_compute_parameters(self.s1, self.g1)
"""
if __name__ == '__main__':
unittest.main()

@ -14,14 +14,14 @@ class TestSamplePath(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
def test_init(self):
s1 = sp.SamplePath(self.importer)
self.assertIsNone(s1.trajectories)
self.assertIsNone(s1.structure)
self.assertFalse(s1.importer.concatenated_samples.empty)
self.assertIsNone(s1.total_variables_count)
self.assertFalse(s1._importer.concatenated_samples.empty)
self.assertIsNone(s1._total_variables_count)
def test_build_trajectories(self):
s1 = sp.SamplePath(self.importer)
@ -32,7 +32,7 @@ class TestSamplePath(unittest.TestCase):
s1 = sp.SamplePath(self.importer)
s1.build_structure()
self.assertIsInstance(s1.structure, st.Structure)
self.assertEqual(s1.total_variables_count, len(s1.importer.sorter))
self.assertEqual(s1._total_variables_count, len(s1._importer.sorter))
if __name__ == '__main__':
unittest.main()

@ -59,17 +59,17 @@ class TestSetOfCims(unittest.TestCase):
state_transition_matrix = np.random.randint(1, 10000, (3, 3))
state_res_times_list.append(state_res_times)
transition_matrices_list.append(state_transition_matrix)
sofc1.build_cims(state_res_times_list, transition_matrices_list)
sofc1.build_cims(np.array(state_res_times_list), np.array(transition_matrices_list))
for length_of_mask in range(3):
for mask in list(itertools.permutations([True, False],r=length_of_mask)):
m = np.array(mask)
for parent_value in range(self.possible_cardinalities[0]):
cims = sofc1.filter_cims_with_mask(m, [parent_value])
if length_of_mask == 0 or length_of_mask == 1:
self.assertTrue(np.array_equal(sofc1.actual_cims, cims))
self.assertTrue(np.array_equal(sofc1._actual_cims, cims))
else:
indxs = self.another_filtering_method(p_combs, m, [parent_value])
self.assertTrue(np.array_equal(cims, sofc1.actual_cims[indxs]))
self.assertTrue(np.array_equal(cims, sofc1._actual_cims[indxs]))
def aux_test_build_cims(self, node_id, p_values, node_states, p_combs):
state_res_times_list = []
@ -81,23 +81,23 @@ class TestSetOfCims(unittest.TestCase):
state_transition_matrix = np.random.randint(1, 10000, (node_states, node_states))
state_res_times_list.append(state_res_times)
transition_matrices_list.append(state_transition_matrix)
so1.build_cims(state_res_times_list, transition_matrices_list)
so1.build_cims(np.array(state_res_times_list), np.array(transition_matrices_list))
self.assertEqual(len(state_res_times_list), so1.get_cims_number())
self.assertIsInstance(so1.actual_cims, np.ndarray)
self.assertIsNone(so1.transition_matrices)
self.assertIsNone(so1.state_residence_times)
self.assertIsInstance(so1._actual_cims, np.ndarray)
self.assertIsNone(so1._transition_matrices)
self.assertIsNone(so1._state_residence_times)
def aux_test_init(self, node_id, parents_states_number, node_states_number, p_combs):
sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number, p_combs)
self.assertEqual(sofcims.node_id, node_id)
self.assertTrue(np.array_equal(sofcims.p_combs, p_combs))
self.assertTrue(np.array_equal(sofcims.parents_states_number, parents_states_number))
self.assertEqual(sofcims.node_states_number, node_states_number)
self.assertFalse(sofcims.actual_cims)
self.assertEqual(sofcims.state_residence_times.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims.state_residence_times[0]),node_states_number)
self.assertEqual(sofcims.transition_matrices.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims.transition_matrices[0][0]), node_states_number)
self.assertEqual(sofcims._node_id, node_id)
self.assertTrue(np.array_equal(sofcims._p_combs, p_combs))
self.assertTrue(np.array_equal(sofcims._parents_states_number, parents_states_number))
self.assertEqual(sofcims._node_states_number, node_states_number)
self.assertFalse(sofcims._actual_cims)
self.assertEqual(sofcims._state_residence_times.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims._state_residence_times[0]), node_states_number)
self.assertEqual(sofcims._transition_matrices.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims._transition_matrices[0][0]), node_states_number)
def aux_test_indexes_converter(self, node_id, parents_states_number, node_states_number):
sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number)

@ -21,7 +21,7 @@ class TestStructureEstimator(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 3)
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()
cls.s1.build_structure()
@ -30,14 +30,14 @@ class TestStructureEstimator(unittest.TestCase):
exp_alfa = 0.1
chi_alfa = 0.1
se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa)
self.assertEqual(self.s1, se1.sample_path)
self.assertTrue(np.array_equal(se1.nodes, np.array(self.s1.structure.nodes_labels)))
self.assertTrue(np.array_equal(se1.nodes_indxs, self.s1.structure.nodes_indexes))
self.assertTrue(np.array_equal(se1.nodes_vals, self.s1.structure.nodes_values))
self.assertEqual(se1.exp_test_sign, exp_alfa)
self.assertEqual(se1.chi_test_alfa, chi_alfa)
self.assertIsInstance(se1.complete_graph, nx.DiGraph)
self.assertIsInstance(se1.cache, ch.Cache)
self.assertEqual(self.s1, se1._sample_path)
self.assertTrue(np.array_equal(se1._nodes, np.array(self.s1.structure.nodes_labels)))
self.assertTrue(np.array_equal(se1._nodes_indxs, self.s1.structure.nodes_indexes))
self.assertTrue(np.array_equal(se1._nodes_vals, self.s1.structure.nodes_values))
self.assertEqual(se1._exp_test_sign, exp_alfa)
self.assertEqual(se1._chi_test_alfa, chi_alfa)
self.assertIsInstance(se1._complete_graph, nx.DiGraph)
self.assertIsInstance(se1._cache, ch.Cache)
def test_build_complete_graph(self):
exp_alfa = 0.1
@ -76,17 +76,18 @@ class TestStructureEstimator(unittest.TestCase):
lp_wrapper = lp(se1.ctpc_algorithm)
lp_wrapper()
lp.print_stats()
print(se1.complete_graph.edges)
print(se1._complete_graph.edges)
print(self.s1.structure.edges)
for ed in self.s1.structure.edges:
self.assertIn(tuple(ed), se1.complete_graph.edges)
self.assertIn(tuple(ed), se1._complete_graph.edges)
tuples_edges = [tuple(rec) for rec in self.s1.structure.edges]
spurious_edges = []
for ed in se1.complete_graph.edges:
for ed in se1._complete_graph.edges:
if not(ed in tuples_edges):
spurious_edges.append(ed)
print("Spurious Edges:",spurious_edges)
se1.save_results()
print("Adj Matrix:", nx.adj_matrix(se1._complete_graph).toarray().astype(bool))
#se1.save_results()
def test_memory(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
@ -95,5 +96,6 @@ class TestStructureEstimator(unittest.TestCase):
mem = current_process.memory_info().rss
print("Average Memory Usage in MB:", mem / 10**6)
if __name__ == '__main__':
unittest.main()