unimib-thesis-mad
/
PyCTBN
Archived
1
0
Fork 0
Code Releases Activity

Refactor class members as protected;Refactors on comments and methods return types

Browse Source
parallel_struct_est
philpMartin 4 years ago
parent 59cda0d3d9
commit 5321999e5e
20 changed files with 517 additions and 607 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 10
      main_package/basic_main.py
  2. 23
      main_package/classes/abstract_importer.py
  3. 26
      main_package/classes/cache.py
  4. 11
      main_package/classes/conditional_intensity_matrix.py
  5. 88
      main_package/classes/json_importer.py
  6. 198
      main_package/classes/network_graph.py
  7. 106
      main_package/classes/parameters_estimator.py
  8. 38
      main_package/classes/sample_path.py
  9. 110
      main_package/classes/set_of_cims.py
  10. 2
      main_package/classes/simple_cvs_importer.py
  11. 17
      main_package/classes/structure.py
  12. 183
      main_package/classes/structure_estimator.py
  13. 10
      main_package/classes/trajectory.py
  14. 28
      main_package/tests/test_cache.py
  15. 96
      main_package/tests/test_json_importer.py
  16. 60
      main_package/tests/test_networkgraph.py
  17. 48
      main_package/tests/test_parameters_estimator.py
  18. 8
      main_package/tests/test_sample_path.py
  19. 32
      main_package/tests/test_setofcims.py
  20. 28
      main_package/tests/test_structure_estimator.py

10
main_package/basic_main.py
Unescape View File

@ -6,7 +6,6 @@ sys.path.append("./classes/")
import network_graph as ng import network_graph as ng
import sample_path as sp import sample_path as sp
import set_of_cims as sofc
import parameters_estimator as pe import parameters_estimator as pe
import json_importer as ji import json_importer as ji
@ -14,20 +13,23 @@ import json_importer as ji
def main(): def main():
read_files = glob.glob(os.path.join('./data', "*.json")) #Take all json files in this dir read_files = glob.glob(os.path.join('./data', "*.json")) #Take all json files in this dir
#import data #import data
importer = ji.JsonImporter(read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') importer = ji.JsonImporter(read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 1)
#Create a SamplePath Obj #Create a SamplePath Obj
s1 = sp.SamplePath(importer) s1 = sp.SamplePath(importer)
#Build The trajectries and the structural infos #Build The trajectries and the structural infos
s1.build_trajectories() s1.build_trajectories()
s1.build_structure() s1.build_structure()
print(s1.structure.edges)
print(s1.structure.nodes_values)
#From The Structure Object build the Graph #From The Structure Object build the Graph
g = ng.NetworkGraph(s1.structure) g = ng.NetworkGraph(s1.structure)
#Select a node you want to estimate the parameters #Select a node you want to estimate the parameters
node = g.nodes[1] node = g.nodes[1]
#Init the graph specifically for THIS node print("NOde", node)
#Init the _graph specifically for THIS node
g.fast_init(node) g.fast_init(node)
#Use SamplePath and Grpah to create a ParametersEstimator Object #Use SamplePath and Grpah to create a ParametersEstimator Object
p1 = pe.ParametersEstimator(s1, g) p1 = pe.ParametersEstimator(s1.trajectories, g)
#Init the peEst specifically for THIS node #Init the peEst specifically for THIS node
p1.fast_init(node) p1.fast_init(node)
#Compute the parameters #Compute the parameters

23
main_package/classes/abstract_importer.py
Unescape View File

@ -7,21 +7,21 @@ class AbstractImporter(ABC):
""" """
Abstract class that exposes all the necessary methods to process the trajectories and the net structure. Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
:file_path: the file path :_file_path: the file path
:_concatenated_samples: the concatenation of all the processed trajectories :_concatenated_samples: the concatenation of all the processed trajectories
:df_structure: Dataframe containing the structure of the network (edges) :_df_structure: Dataframe containing the structure of the network (edges)
:df_variables: Dataframe containing the nodes cardinalities :_df_variables: Dataframe containing the nodes cardinalities
:df_concatenated_samples: the concatenation and processing of all the trajectories present :_df_concatenated_samples: the concatenation and processing of all the trajectories present
in the list df_samples list in the list df_samples list
:sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples :_sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples
""" """
def __init__(self, file_path: str): def __init__(self, file_path: str):
""" """
Parameters: Parameters:
:file_path: the path to the file containing the data :_file_path: the path to the file containing the data
""" """
self.file_path = file_path self._file_path = file_path
self._df_variables = None self._df_variables = None
self._df_structure = None self._df_structure = None
self._concatenated_samples = None self._concatenated_samples = None
@ -71,7 +71,6 @@ class AbstractImporter(ABC):
pre: the Dataframe sample_frame has to follow the column structure of this header: pre: the Dataframe sample_frame has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values] Header of sample_frame = [Time | Variable values]
""" """
#sample_frame[time_header_label] = sample_frame[time_header_label].diff().shift(-1)
sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1) sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1)
shifted_cols = sample_frame[columns_header].shift(-1).fillna(0).astype('int32') shifted_cols = sample_frame[columns_header].shift(-1).fillna(0).astype('int32')
shifted_cols.columns = shifted_cols_header shifted_cols.columns = shifted_cols_header
@ -81,7 +80,7 @@ class AbstractImporter(ABC):
def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List): def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List):
""" """
Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list df_samples_list. Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list _df_samples_list.
Concatenates the result in the dataframe concatanated_samples Concatenates the result in the dataframe concatanated_samples
Parameters: Parameters:
time_header_label: the label of the time column time_header_label: the label of the time column
@ -103,9 +102,7 @@ class AbstractImporter(ABC):
complete_header = self._sorter[:] complete_header = self._sorter[:]
complete_header.insert(0,'Time') complete_header.insert(0,'Time')
complete_header.extend(shifted_cols_header) complete_header.extend(shifted_cols_header)
#print("Complete Header", complete_header)
self._concatenated_samples = self._concatenated_samples[complete_header] self._concatenated_samples = self._concatenated_samples[complete_header]
#print("Concat Samples",self._concatenated_samples)
def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List: def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List:
""" """
@ -143,3 +140,7 @@ class AbstractImporter(ABC):
@property @property
def sorter(self) -> typing.List: def sorter(self) -> typing.List:
return self._sorter return self._sorter
@property
def file_path(self) -> str:
return self._file_path

26
main_package/classes/cache.py
Unescape View File

@ -7,28 +7,28 @@ class Cache:
""" """
This class has the role of a cache for SetOfCIMS of a test node that have been already computed during the ctpc algorithm. This class has the role of a cache for SetOfCIMS of a test node that have been already computed during the ctpc algorithm.
:list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related :_list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related
:actual_cache: a list of setOfCims objects :_actual_cache: a list of setOfCims objects
""" """
def __init__(self): def __init__(self):
self.list_of_sets_of_parents = [] self._list_of_sets_of_parents = []
self.actual_cache = [] self._actual_cache = []
def find(self, parents_comb: typing.Set): #typing.Union[typing.Set, str] def find(self, parents_comb: typing.Set) -> sofc.SetOfCims:
""" """
Tries to find in cache given the symbolic parents combination parents_comb the SetOfCims related to that parents_comb. Tries to find in cache given the symbolic parents combination parents_comb the SetOfCims related to that parents_comb.
Parameters: Parameters:
parents_comb: the parents related to that SetOfCims parents_comb: the parents related to that SetOfCims
Returns: Returns:
A SetOfCims object if the parents_comb index is found in list_of_sets_of_parents. A SetOfCims object if the parents_comb index is found in _list_of_sets_of_parents.
None otherwise. None otherwise.
""" """
try: try:
#print("Cache State:", self.list_of_sets_of_indxs) #print("Cache State:", self.list_of_sets_of_indxs)
#print("Look For:", parents_comb) #print("Look For:", parents_comb)
result = self.actual_cache[self.list_of_sets_of_parents.index(parents_comb)] result = self._actual_cache[self._list_of_sets_of_parents.index(parents_comb)]
print("CACHE HIT!!!!", parents_comb) print("CACHE HIT!!!!", parents_comb)
return result return result
except ValueError: except ValueError:
@ -36,7 +36,7 @@ class Cache:
def put(self, parents_comb: typing.Union[typing.Set, str], socim: sofc.SetOfCims): def put(self, parents_comb: typing.Union[typing.Set, str], socim: sofc.SetOfCims):
""" """
Place in cache the SetOfCims object, and the related sybolyc index parents_comb in list_of_sets_of_parents Place in cache the SetOfCims object, and the related sybolyc index parents_comb in _list_of_sets_of_parents
Parameters: Parameters:
parents_comb: the symbolic set index parents_comb: the symbolic set index
@ -45,9 +45,9 @@ class Cache:
Returns: Returns:
void void
""" """
#print("Putting in cache:", parents_comb) #print("Putting in _cache:", parents_comb)
self.list_of_sets_of_parents.append(parents_comb) self._list_of_sets_of_parents.append(parents_comb)
self.actual_cache.append(socim) self._actual_cache.append(socim)
def clear(self): def clear(self):
""" """
@ -58,5 +58,5 @@ class Cache:
Returns: Returns:
void void
""" """
del self.list_of_sets_of_parents[:] del self._list_of_sets_of_parents[:]
del self.actual_cache[:] del self._actual_cache[:]

11
main_package/classes/conditional_intensity_matrix.py
Unescape View File

@ -11,6 +11,11 @@ class ConditionalIntensityMatrix:
:_cim: the actual cim of the node :_cim: the actual cim of the node
""" """
def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array): def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array):
"""
Parameters:
:_state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix
"""
self._state_residence_times = state_residence_times self._state_residence_times = state_residence_times
self._state_transition_matrix = state_transition_matrix self._state_transition_matrix = state_transition_matrix
self._cim = self.state_transition_matrix.astype(np.float64) self._cim = self.state_transition_matrix.astype(np.float64)
@ -28,15 +33,15 @@ class ConditionalIntensityMatrix:
self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T
@property @property
def state_residence_times(self): def state_residence_times(self) -> np.ndarray:
return self._state_residence_times return self._state_residence_times
@property @property
def state_transition_matrix(self): def state_transition_matrix(self) -> np.ndarray:
return self._state_transition_matrix return self._state_transition_matrix
@property @property
def cim(self): def cim(self) -> np.ndarray:
return self._cim return self._cim
def __repr__(self): def __repr__(self):

88
main_package/classes/json_importer.py
Unescape View File

@ -16,31 +16,33 @@ class JsonImporter(ai.AbstractImporter):
|_ samples |_ samples
|_ variabels |_ variabels
:file_path: the path of the file that contains tha data to be imported :file_path: the path of the file that contains tha data to be imported
:samples_label: the reference key for the samples in the trajectories :_samples_label: the reference key for the samples in the trajectories
:structure_label: the reference key for the structure of the network data :_structure_label: the reference key for the structure of the network data
:variables_label: the reference key for the cardinalites of the nodes data :_variables_label: the reference key for the cardinalites of the nodes data
:time_key: the key used to identify the timestamps in each trajectory :_time_key: the key used to identify the timestamps in each trajectory
:variables_key: the key used to identify the names of the variables in the net :_variables_key: the key used to identify the names of the variables in the net
:df_samples_list: a Dataframe list in which every df contains a trajectory :_df_samples_list: a Dataframe list in which every df contains a trajectory
""" """
def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str, def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str,
variables_key: str): variables_key: str, array_indx: int):
""" """
Parameters: Parameters:
file_path: the path of the file that contains tha data to be imported :file_path: the path of the file that contains tha data to be imported
:samples_label: the reference key for the samples in the trajectories :_samples_label: the reference key for the samples in the trajectories
:structure_label: the reference key for the structure of the network data :_structure_label: the reference key for the structure of the network data
:variables_label: the reference key for the cardinalites of the nodes data :_variables_label: the reference key for the cardinalites of the nodes data
:time_key: the key used to identify the timestamps in each trajectory :_time_key: the key used to identify the timestamps in each trajectory
:variables_key: the key used to identify the names of the variables in the net :_variables_key: the key used to identify the names of the variables in the net
""" :_array_indx: the index of the outer json array from which import all the data
self.samples_label = samples_label """
self.structure_label = structure_label self._samples_label = samples_label
self.variables_label = variables_label self._structure_label = structure_label
self.time_key = time_key self._variables_label = variables_label
self.variables_key = variables_key self._time_key = time_key
self.df_samples_list = None self._variables_key = variables_key
self._df_samples_list = None
self._array_indx = array_indx
super(JsonImporter, self).__init__(file_path) super(JsonImporter, self).__init__(file_path)
def import_data(self): def import_data(self):
@ -52,14 +54,14 @@ class JsonImporter(ai.AbstractImporter):
_void _void
""" """
raw_data = self.read_json_file() raw_data = self.read_json_file()
self.df_samples_list = self.import_trajectories(raw_data) self._df_samples_list = self.import_trajectories(raw_data)
self._sorter = self.build_sorter(self.df_samples_list[0]) self._sorter = self.build_sorter(self._df_samples_list[0])
self.compute_row_delta_in_all_samples_frames(self.df_samples_list) self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
self.clear_data_frame_list() self.clear_data_frame_list()
self._df_structure = self.import_structure(raw_data) self._df_structure = self.import_structure(raw_data)
self._df_variables = self.import_variables(raw_data, self._sorter) self._df_variables = self.import_variables(raw_data)
def import_trajectories(self, raw_data: typing.List): def import_trajectories(self, raw_data: typing.List) -> typing.List:
""" """
Imports the trajectories in the list of dicts raw_data. Imports the trajectories in the list of dicts raw_data.
Parameters: Parameters:
@ -67,22 +69,22 @@ class JsonImporter(ai.AbstractImporter):
Returns: Returns:
:List of dataframes containing all the trajectories :List of dataframes containing all the trajectories
""" """
return self.normalize_trajectories(raw_data, 0, self.samples_label) return self.normalize_trajectories(raw_data, self._array_indx, self._samples_label)
def import_structure(self, raw_data: typing.List) -> pd.DataFrame: def import_structure(self, raw_data: typing.List) -> pd.DataFrame:
""" """
Imports in a dataframe the data in the list raw_data at the key structure_label Imports in a dataframe the data in the list raw_data at the key _structure_label
Parameters: Parameters:
:raw_data: the data :raw_data: the data
Returns: Returns:
:Daframe containg the starting node a ending node of every arc of the network :Daframe containg the starting node a ending node of every arc of the network
""" """
return self.one_level_normalizing(raw_data, 0, self.structure_label) return self.one_level_normalizing(raw_data, self._array_indx, self._structure_label)
def import_variables(self, raw_data: typing.List, sorter: typing.List) -> pd.DataFrame: def import_variables(self, raw_data: typing.List) -> pd.DataFrame:
""" """
Imports the data in raw_data at the key variables_label. Imports the data in raw_data at the key _variables_label.
Sorts the row of the dataframe df_variables using the list sorter. Sorts the row of the dataframe df_variables using the list sorter.
Parameters: Parameters:
@ -91,16 +93,7 @@ class JsonImporter(ai.AbstractImporter):
Returns: Returns:
:Datframe containg the variables simbolic labels and their cardinalities :Datframe containg the variables simbolic labels and their cardinalities
""" """
return self.one_level_normalizing(raw_data, 0, self.variables_label) return self.one_level_normalizing(raw_data, self._array_indx, self._variables_label)
#TODO Usando come Pre-requisito l'ordinamento del frame _df_variables uguale a quello presente in
#TODO self _sorter questo codice risulta inutile
"""self._df_variables[self.variables_key] = self._df_variables[self.variables_key].astype("category")
self._df_variables[self.variables_key] = self._df_variables[self.variables_key].cat.set_categories(sorter)
self._df_variables = self._df_variables.sort_values([self.variables_key])
self._df_variables.reset_index(inplace=True)
self._df_variables.drop('index', axis=1, inplace=True)
#print("Var Frame", self._df_variables)
"""
def read_json_file(self) -> typing.List: def read_json_file(self) -> typing.List:
""" """
@ -112,7 +105,7 @@ class JsonImporter(ai.AbstractImporter):
:data: the contents of the json file :data: the contents of the json file
""" """
with open(self.file_path) as f: with open(self._file_path) as f:
data = json.load(f) data = json.load(f)
return data return data
@ -130,7 +123,7 @@ class JsonImporter(ai.AbstractImporter):
""" """
return pd.DataFrame(raw_data[indx][key]) return pd.DataFrame(raw_data[indx][key])
def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str): def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str) -> typing.List:
""" """
Extracts the traj in raw_data at the index index at the key trajectories key. Extracts the traj in raw_data at the index index at the key trajectories key.
@ -145,28 +138,25 @@ class JsonImporter(ai.AbstractImporter):
smps = raw_data[indx][trajectories_key] smps = raw_data[indx][trajectories_key]
df_samples_list = [dataframe(sample) for sample in smps] df_samples_list = [dataframe(sample) for sample in smps]
return df_samples_list return df_samples_list
#columns_header = list(self.df_samples_list[0].columns.values)
#columns_header.remove(self.time_key)
#self._sorter = columns_header
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List: def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
""" """
Implements the abstract method build_sorter for this dataset Implements the abstract method build_sorter for this dataset
""" """
columns_header = list(sample_frame.columns.values) columns_header = list(sample_frame.columns.values)
columns_header.remove(self.time_key) columns_header.remove(self._time_key)
return columns_header return columns_header
def clear_data_frame_list(self): def clear_data_frame_list(self):
""" """
Removes all values present in the dataframes in the list df_samples_list Removes all values present in the dataframes in the list _df_samples_list
Parameters: Parameters:
:void :void
Returns: Returns:
:void :void
""" """
for indx in range(len(self.df_samples_list)): for indx in range(len(self._df_samples_list)):
self.df_samples_list[indx] = self.df_samples_list[indx].iloc[0:0] self._df_samples_list[indx] = self._df_samples_list[indx].iloc[0:0]
def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict: def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict:
""" """

198
main_package/classes/network_graph.py
Unescape View File

@ -1,37 +1,42 @@
import typing import typing
import structure as st
import networkx as nx import networkx as nx
import numpy as np import numpy as np
class NetworkGraph: class NetworkGraph:
""" """
Abstracts the infos contained in the Structure class in the form of a directed graph. Abstracts the infos contained in the Structure class in the form of a directed _graph.
Has the task of creating all the necessary filtering structures for parameters estimation Has the task of creating all the necessary filtering structures for parameters estimation
:graph_struct: the Structure object from which infos about the net will be extracted :_graph_struct: the Structure object from which infos about the net will be extracted
:graph: directed graph :_graph: directed _graph
:nodes_labels: the symbolic names of the variables :nodes_labels: the symbolic names of the variables
:nodes_indexes: the indexes of the nodes :nodes_indexes: the indexes of the nodes
:nodes_values: the cardinalites of the nodes :nodes_values: the cardinalites of the nodes
:aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every :_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every
node in the net node in the net
:_fancy_indexing: the indexes of every parent of every node in the net :_fancy_indexing: the indexes of every parent of every node in the net
:_time_scalar_indexing_structure: the indexing structure for state res time estimation :_time_scalar_indexing_structure: the indexing structure for state res time estimation
:_transition_scalar_indexing_structure: the indexing structure for transition computation :_transition_scalar_indexing_structure: the indexing structure for transition computation
:_time_filtering: the columns filtering structure used in the computation of the state res times :_time_filtering: the columns filtering structure used in the computation of the state res times
:_transition_filtering: the columns filtering structure used in the computation of the transition from one state to another :_transition_filtering: the columns filtering structure used in the computation of the transition
from one state to another
:self._p_combs_structure: all the possible parents states combination for every node in the net :self._p_combs_structure: all the possible parents states combination for every node in the net
""" """
def __init__(self, graph_struct): def __init__(self, graph_struct: st.Structure):
self.graph_struct = graph_struct """
self.graph = nx.DiGraph() Parameters:
self._nodes_indexes = self.graph_struct.nodes_indexes :graph_struct:the Structure object from which infos about the net will be extracted
self._nodes_labels = self.graph_struct.nodes_labels """
self._nodes_values = self.graph_struct.nodes_values self._graph_struct = graph_struct
self.aggregated_info_about_nodes_parents = None self._graph = nx.DiGraph()
self._nodes_indexes = self._graph_struct.nodes_indexes
self._nodes_labels = self._graph_struct.nodes_labels
self._nodes_values = self._graph_struct.nodes_values
self._aggregated_info_about_nodes_parents = None
self._fancy_indexing = None self._fancy_indexing = None
self._time_scalar_indexing_structure = None self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None self._transition_scalar_indexing_structure = None
@ -39,16 +44,6 @@ class NetworkGraph:
self._transition_filtering = None self._transition_filtering = None
self._p_combs_structure = None self._p_combs_structure = None
def init_graph(self):
self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.edges)
self.aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures()
self.build_time_columns_filtering_structure()
self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
def fast_init(self, node_id: str): def fast_init(self, node_id: str):
""" """
Initializes all the necessary structures for parameters estimation of the node identified by the label node_id Initializes all the necessary structures for parameters estimation of the node identified by the label node_id
@ -58,11 +53,11 @@ class NetworkGraph:
void void
""" """
self.add_nodes(self._nodes_labels) self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.edges) self.add_edges(self._graph_struct.edges)
self.aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id) self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
self._fancy_indexing = self.aggregated_info_about_nodes_parents[1] self._fancy_indexing = self._aggregated_info_about_nodes_parents[1]
p_indxs = self._fancy_indexing p_indxs = self._fancy_indexing
p_vals = self.aggregated_info_about_nodes_parents[2] p_vals = self._aggregated_info_about_nodes_parents[2]
self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id, self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id,
p_vals) p_vals)
self._transition_scalar_indexing_structure = self.build_transition_scalar_indexing_structure_for_a_node(node_id, self._transition_scalar_indexing_structure = self.build_transition_scalar_indexing_structure_for_a_node(node_id,
@ -74,33 +69,33 @@ class NetworkGraph:
def add_nodes(self, list_of_nodes: typing.List): def add_nodes(self, list_of_nodes: typing.List):
""" """
Adds the nodes to the graph contained in the list of nodes list_of_nodes. Adds the nodes to the _graph contained in the list of nodes list_of_nodes.
Sets all the properties that identify a nodes (index, positional index, cardinality) Sets all the properties that identify a nodes (index, positional index, cardinality)
Parameters: Parameters:
list_of_nodes: the nodes to add to graph list_of_nodes: the nodes to add to _graph
Returns: Returns:
void void
""" """
nodes_indxs = self._nodes_indexes nodes_indxs = self._nodes_indexes
nodes_vals = self.graph_struct.nodes_values nodes_vals = self._graph_struct.nodes_values
pos = 0 pos = 0
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals): for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self.graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos) self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1 pos += 1
def add_edges(self, list_of_edges: typing.List): def add_edges(self, list_of_edges: typing.List):
""" """
Add the edges to the graph contained in the list list_of_edges. Add the edges to the _graph contained in the list list_of_edges.
Parameters: Parameters:
list_of_edges list_of_edges
Returns: Returns:
void void
""" """
self.graph.add_edges_from(list_of_edges) self._graph.add_edges_from(list_of_edges)
def get_ordered_by_indx_set_of_parents(self, node: str): def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple:
""" """
Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
(parents_labels, parents_indexes, parents_cardinalities). (parents_labels, parents_indexes, parents_cardinalities).
@ -121,22 +116,6 @@ class NetworkGraph:
p_values = [self.get_states_number(node) for node in sorted_parents] p_values = [self.get_states_number(node) for node in sorted_parents]
return (sorted_parents, p_indxes, p_values) return (sorted_parents, p_indxes, p_values)
def get_ord_set_of_par_of_all_nodes(self):
get_ordered_by_indx_set_of_parents = self.get_ordered_by_indx_set_of_parents
result = [get_ordered_by_indx_set_of_parents(node) for node in self._nodes_labels]
return result
def get_ordered_by_indx_parents_values_for_all_nodes(self):
pars_values = [i[2] for i in self.aggregated_info_about_nodes_parents]
return pars_values
def build_fancy_indexing_structure(self, start_indx):
if start_indx > 0:
pass
else:
fancy_indx = [i[1] for i in self.aggregated_info_about_nodes_parents]
return fancy_indx
def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray: def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray:
""" """
Builds an indexing structure for the computation of state residence times values. Builds an indexing structure for the computation of state residence times values.
@ -153,8 +132,8 @@ class NetworkGraph:
T_vector = T_vector.cumprod().astype(np.int) T_vector = T_vector.cumprod().astype(np.int)
return T_vector return T_vector
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) \
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray: -> np.ndarray:
""" """
Builds an indexing structure for the computation of state transitions values. Builds an indexing structure for the computation of state transitions values.
@ -194,7 +173,7 @@ class NetworkGraph:
Returns: Returns:
a numpy array a numpy array
""" """
nodes_number = self.graph_struct.total_variables_number nodes_number = self._graph_struct.total_variables_number
return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int) return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray: def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray:
@ -219,81 +198,108 @@ class NetworkGraph:
parents_comb = np.array([[]], dtype=np.int) parents_comb = np.array([[]], dtype=np.int)
return parents_comb return parents_comb
def build_time_columns_filtering_structure(self):
nodes_indxs = self._nodes_indexes
self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)]
def build_transition_columns_filtering_structure(self):
nodes_number = self.graph_struct.total_variables_number
nodes_indxs = self._nodes_indexes
self._transition_filtering = [np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
for node_indx, p_indxs in zip(nodes_indxs,
self._fancy_indexing)]
def build_scalar_indexing_structures(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
build_transition_scalar_indexing_structure_for_a_node = self.build_transition_scalar_indexing_structure_for_a_node
build_time_scalar_indexing_structure_for_a_node = self.build_time_scalar_indexing_structure_for_a_node
aggr = [(build_transition_scalar_indexing_structure_for_a_node(node_id, p_vals),
build_time_scalar_indexing_structure_for_a_node(node_id, p_vals))
for node_id, p_vals in
zip(self._nodes_labels,
parents_values_for_all_nodes)]
self._transition_scalar_indexing_structure = [i[0] for i in aggr]
self._time_scalar_indexing_structure = [i[1] for i in aggr]
def build_p_combs_structure(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs_struct = [self.build_p_comb_structure_for_a_node(p_vals) for p_vals in parents_values_for_all_nodes]
return p_combs_struct
def get_parents_by_id(self, node_id): def get_parents_by_id(self, node_id):
return list(self.graph.predecessors(node_id)) return list(self._graph.predecessors(node_id))
def get_states_number(self, node_id): def get_states_number(self, node_id):
return self.graph.nodes[node_id]['val'] return self._graph.nodes[node_id]['val']
def get_node_indx(self, node_id): def get_node_indx(self, node_id):
return nx.get_node_attributes(self.graph, 'indx')[node_id] return nx.get_node_attributes(self._graph, 'indx')[node_id]
def get_positional_node_indx(self, node_id): def get_positional_node_indx(self, node_id):
return self.graph.nodes[node_id]['pos_indx'] return self._graph.nodes[node_id]['pos_indx']
@property @property
def nodes(self): def nodes(self) -> typing.List:
return self._nodes_labels return self._nodes_labels
@property @property
def edges(self): def edges(self) -> typing.List:
return list(self.graph.edges) return list(self._graph.edges)
@property @property
def nodes_indexes(self): def nodes_indexes(self) -> np.ndarray:
return self._nodes_indexes return self._nodes_indexes
@property @property
def nodes_values(self): def nodes_values(self) -> np.ndarray:
return self._nodes_values return self._nodes_values
@property @property
def time_scalar_indexing_strucure(self): def time_scalar_indexing_strucure(self) -> np.ndarray:
return self._time_scalar_indexing_structure return self._time_scalar_indexing_structure
@property @property
def time_filtering(self): def time_filtering(self) -> np.ndarray:
return self._time_filtering return self._time_filtering
@property @property
def transition_scalar_indexing_structure(self): def transition_scalar_indexing_structure(self) -> np.ndarray:
return self._transition_scalar_indexing_structure return self._transition_scalar_indexing_structure
@property @property
def transition_filtering(self): def transition_filtering(self) -> np.ndarray:
return self._transition_filtering return self._transition_filtering
@property @property
def p_combs(self): def p_combs(self) -> np.ndarray:
return self._p_combs_structure return self._p_combs_structure
"""##############These Methods are actually unused but could become useful in the near future################"""
def init_graph(self):
self.add_nodes(self._nodes_labels)
self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures()
self.build_time_columns_filtering_structure()
self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
def build_time_columns_filtering_structure(self):
nodes_indxs = self._nodes_indexes
self._time_filtering = [np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
for node_indx, p_indxs in zip(nodes_indxs, self._fancy_indexing)]
def build_transition_columns_filtering_structure(self):
nodes_number = self._graph_struct.total_variables_number
nodes_indxs = self._nodes_indexes
self._transition_filtering = [np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
for node_indx, p_indxs in zip(nodes_indxs,
self._fancy_indexing)]
def build_scalar_indexing_structures(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
build_transition_scalar_indexing_structure_for_a_node = \
self.build_transition_scalar_indexing_structure_for_a_node
build_time_scalar_indexing_structure_for_a_node = self.build_time_scalar_indexing_structure_for_a_node
aggr = [(build_transition_scalar_indexing_structure_for_a_node(node_id, p_vals),
build_time_scalar_indexing_structure_for_a_node(node_id, p_vals))
for node_id, p_vals in
zip(self._nodes_labels,
parents_values_for_all_nodes)]
self._transition_scalar_indexing_structure = [i[0] for i in aggr]
self._time_scalar_indexing_structure = [i[1] for i in aggr]
def build_p_combs_structure(self):
parents_values_for_all_nodes = self.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs_struct = [self.build_p_comb_structure_for_a_node(p_vals) for p_vals in parents_values_for_all_nodes]
return p_combs_struct
def get_ord_set_of_par_of_all_nodes(self):
get_ordered_by_indx_set_of_parents = self.get_ordered_by_indx_set_of_parents
result = [get_ordered_by_indx_set_of_parents(node) for node in self._nodes_labels]
return result
def get_ordered_by_indx_parents_values_for_all_nodes(self):
pars_values = [i[2] for i in self._aggregated_info_about_nodes_parents]
return pars_values
def build_fancy_indexing_structure(self, start_indx):
if start_indx > 0:
pass
else:
fancy_indx = [i[1] for i in self._aggregated_info_about_nodes_parents]
return fancy_indx

106
main_package/classes/parameters_estimator.py
Unescape View File

@ -2,7 +2,7 @@
import numpy as np import numpy as np
import network_graph as ng import network_graph as ng
import sample_path as sp import trajectory as tr
import set_of_cims as sofc import set_of_cims as sofc
import sets_of_cims_container as acims import sets_of_cims_container as acims
@ -10,18 +10,24 @@ import sets_of_cims_container as acims
class ParametersEstimator: class ParametersEstimator:
""" """
Has the task of computing the cims of particular node given the trajectories in samplepath and the net structure Has the task of computing the cims of particular node given the trajectories in samplepath and the net structure
in the graph net_graph in the graph _net_graph
:sample_path: the container of the trajectories :trajectories: the trajectories
:net_graph: the net structure :_net_graph: the net structure
:single_srt_of_cims: the set of cims object that will hold the cims of the node :_single_set_of_cims: the set of cims object that will hold the cims of the node
""" """
def __init__(self, sample_path: sp.SamplePath, net_graph: ng.NetworkGraph): def __init__(self, trajectories: tr.Trajectory, net_graph: ng.NetworkGraph):
self.sample_path = sample_path """
self.net_graph = net_graph Parameters:
self.sets_of_cims_struct = None :trajectories: the trajectories
self.single_set_of_cims = None :_net_graph: the net structure
"""
#self.sample_path = sample_path
self._trajectories = trajectories
self._net_graph = net_graph
#self.sets_of_cims_struct = None
self._single_set_of_cims = None
def fast_init(self, node_id: str): def fast_init(self, node_id: str):
""" """
@ -32,9 +38,9 @@ class ParametersEstimator:
Returns: Returns:
void void
""" """
p_vals = self.net_graph.aggregated_info_about_nodes_parents[2] p_vals = self._net_graph._aggregated_info_about_nodes_parents[2]
node_states_number = self.net_graph.get_states_number(node_id) node_states_number = self._net_graph.get_states_number(node_id)
self.single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self.net_graph.p_combs) self._single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs)
def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims: def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims:
""" """
@ -45,22 +51,22 @@ class ParametersEstimator:
Returns: Returns:
A setOfCims object filled with the computed CIMS A setOfCims object filled with the computed CIMS
""" """
node_indx = self.net_graph.get_node_indx(node_id) node_indx = self._net_graph.get_node_indx(node_id)
state_res_times = self.single_set_of_cims.state_residence_times state_res_times = self._single_set_of_cims._state_residence_times
transition_matrices = self.single_set_of_cims.transition_matrices transition_matrices = self._single_set_of_cims._transition_matrices
trajectory = self.sample_path.trajectories.trajectory #trajectory = self.sample_path.trajectories.trajectory
self.compute_state_res_time_for_node(node_indx, self.sample_path.trajectories.times, self.compute_state_res_time_for_node(node_indx, self._trajectories.times,
trajectory, self._trajectories.trajectory,
self.net_graph.time_filtering, self._net_graph.time_filtering,
self.net_graph.time_scalar_indexing_strucure, self._net_graph.time_scalar_indexing_strucure,
state_res_times) state_res_times)
self.compute_state_transitions_for_a_node(node_indx, self.compute_state_transitions_for_a_node(node_indx,
self.sample_path.trajectories.complete_trajectory, self._trajectories.complete_trajectory,
self.net_graph.transition_filtering, self._net_graph.transition_filtering,
self.net_graph.transition_scalar_indexing_structure, self._net_graph.transition_scalar_indexing_structure,
transition_matrices) transition_matrices)
self.single_set_of_cims.build_cims(state_res_times, transition_matrices) self._single_set_of_cims.build_cims(state_res_times, transition_matrices)
return self.single_set_of_cims return self._single_set_of_cims
def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray, def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray,
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray, T: np.ndarray): cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray, T: np.ndarray):
@ -99,38 +105,44 @@ class ParametersEstimator:
diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])], diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])],
dtype=np.int64) dtype=np.int64)
trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0] trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0]
#print("Trajectory", trajectory)
#print("Step 1", trajectory[:, int(trajectory.shape[1] / 2) + node_indx])
#print("Step 2", trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0)
#print("TrTemp", trj_tmp)
#print("Cols Filter", cols_filter)
#print("Filtered Tr Temp", trj_tmp[:, cols_filter])
#print("Actual Indexing", scalar_indexing / scalar_indexing[0])
#print("PreBins",trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0] )
#print("Bins", np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1))
#print("After BinCount", np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int)))
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int), M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int),
minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2]) minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
M_raveled = M.ravel() M_raveled = M.ravel()
M_raveled[diag_indices] = 0 M_raveled[diag_indices] = 0
M_raveled[diag_indices] = np.sum(M, axis=2).ravel() M_raveled[diag_indices] = np.sum(M, axis=2).ravel()
"""##############These Methods are actually unused but could become useful in the near future################"""
def init_sets_cims_container(self): def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self.net_graph.nodes, self.sets_of_cims_struct = acims.SetsOfCimsContainer(self._net_graph.nodes,
self.net_graph.nodes_values, self._net_graph.nodes_values,
self.net_graph.get_ordered_by_indx_parents_values_for_all_nodes(), self._net_graph.
self.net_graph.p_combs) get_ordered_by_indx_parents_values_for_all_nodes(),
self._net_graph.p_combs)
def compute_parameters(self): def compute_parameters(self):
#print(self.net_graph.get_nodes()) for indx, aggr in enumerate(zip(self._net_graph.nodes, self.sets_of_cims_struct.sets_of_cims)):
#print(self.amalgamated_cims_struct.sets_of_cims) self.compute_state_res_time_for_node(self._net_graph.get_node_indx(aggr[0]), self.sample_path.trajectories.times,
#enumerate(zip(self.net_graph.get_nodes(), self.amalgamated_cims_struct.sets_of_cims))
for indx, aggr in enumerate(zip(self.net_graph.nodes, self.sets_of_cims_struct.sets_of_cims)):
#print(self.net_graph.time_filtering[indx])
#print(self.net_graph.time_scalar_indexing_strucure[indx])
self.compute_state_res_time_for_node(self.net_graph.get_node_indx(aggr[0]), self.sample_path.trajectories.times,
self.sample_path.trajectories.trajectory, self.sample_path.trajectories.trajectory,
self.net_graph.time_filtering[indx], self._net_graph.time_filtering[indx],
self.net_graph.time_scalar_indexing_strucure[indx], self._net_graph.time_scalar_indexing_strucure[indx],
aggr[1].state_residence_times) aggr[1]._state_residence_times)
#print(self.net_graph.transition_filtering[indx]) self.compute_state_transitions_for_a_node(self._net_graph.get_node_indx(aggr[0]),
#print(self.net_graph.transition_scalar_indexing_structure[indx])
self.compute_state_transitions_for_a_node(self.net_graph.get_node_indx(aggr[0]),
self.sample_path.trajectories.complete_trajectory, self.sample_path.trajectories.complete_trajectory,
self.net_graph.transition_filtering[indx], self._net_graph.transition_filtering[indx],
self.net_graph.transition_scalar_indexing_structure[indx], self._net_graph.transition_scalar_indexing_structure[indx],
aggr[1].transition_matrices) aggr[1]._transition_matrices)
aggr[1].build_cims(aggr[1].state_residence_times, aggr[1].transition_matrices) aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices)

38
main_package/classes/sample_path.py
Unescape View File

@ -10,10 +10,10 @@ class SamplePath:
cardinalites. cardinalites.
Has the task of creating the objects that will contain the mentioned data. Has the task of creating the objects that will contain the mentioned data.
:importer: the Importer objects that will import ad process data :_importer: the Importer objects that will import ad process data
:trajectories: the Trajectory object that will contain all the concatenated trajectories :trajectories: the Trajectory object that will contain all the concatenated trajectories
:structure: the Structure Object that will contain all the structurral infos about the net :structure: the Structure Object that will contain all the structurral infos about the net
:total_variables_count: the number of variables in the net :_total_variables_count: the number of variables in the net
""" """
def __init__(self, importer: imp.AbstractImporter): def __init__(self, importer: imp.AbstractImporter):
@ -21,11 +21,11 @@ class SamplePath:
Parameters: Parameters:
:importer: the Importer objects that will import ad process data :importer: the Importer objects that will import ad process data
""" """
self.importer = importer self._importer = importer
self._trajectories = None self._trajectories = None
self._structure = None self._structure = None
self.total_variables_count = None self._total_variables_count = None
self.importer.import_data() self._importer.import_data()
def build_trajectories(self): def build_trajectories(self):
""" """
@ -37,12 +37,12 @@ class SamplePath:
Returns: Returns:
:void :void
""" """
#self.importer.import_data() #self._importer.import_data()
self._trajectories = \ self._trajectories = \
tr.Trajectory(self.importer.build_list_of_samples_array(self.importer.concatenated_samples), tr.Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples),
len(self.importer.sorter) + 1) len(self._importer.sorter) + 1)
#self.trajectories.append(trajectory) #self.trajectories.append(trajectory)
self.importer.clear_concatenated_frame() self._importer.clear_concatenated_frame()
def build_structure(self): def build_structure(self):
""" """
@ -52,19 +52,16 @@ class SamplePath:
Returns: Returns:
:void :void
""" """
if self.importer.sorter != self.importer.variables.iloc[:, 0].to_list(): if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list():
raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!") raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!")
self.total_variables_count = len(self.importer.sorter) self._total_variables_count = len(self._importer.sorter)
#labels = self.importer.variables[self.importer.variables_key].to_list() labels = self._importer.variables.iloc[:, 0].to_list()
#print("SAMPLE PATH LABELS",labels) indxs = self._importer.variables.index.to_numpy()
#print(self.importer.variables) vals = self._importer.variables.iloc[:, 1].to_numpy()
labels = self.importer.variables.iloc[:, 0].to_list() edges = list(self._importer.structure.to_records(index=False))
indxs = self.importer.variables.index.to_numpy()
vals = self.importer.variables.iloc[:, 1].to_numpy()
edges = list(self.importer.structure.to_records(index=False))
self._structure = st.Structure(labels, indxs, vals, edges, self._structure = st.Structure(labels, indxs, vals, edges,
self.total_variables_count) self._total_variables_count)
@property @property
def trajectories(self) -> tr.Trajectory: def trajectories(self) -> tr.Trajectory:
@ -74,8 +71,9 @@ class SamplePath:
def structure(self) -> st.Structure: def structure(self) -> st.Structure:
return self._structure return self._structure
@property
def total_variables_count(self): def total_variables_count(self):
return self.total_variables_count return self._total_variables_count

110
main_package/classes/set_of_cims.py
Unescape View File

@ -7,26 +7,33 @@ import conditional_intensity_matrix as cim
class SetOfCims: class SetOfCims:
""" """
Aggregates all the CIMS of the node identified by the label node_id. Aggregates all the CIMS of the node identified by the label _node_id.
:node_id: the node label
:parents_states_number: the cardinalities of the parents
:node_states_number: the caridinality of the node
:p_combs: the relative p_comb structure
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
:state_residence_time: matrix containing all the state residence time vectors for the node :state_residence_time: matrix containing all the state residence time vectors for the node
:transition_matrices: matrix containing all the transition matrices for the node :_transition_matrices: matrix containing all the transition matrices for the node
:actaul_cims: the cims of the node :_actual_cims: the cims of the node
""" """
def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray): def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray):
self.node_id = node_id """
self.parents_states_number = parents_states_number Parameters:
self.node_states_number = node_states_number :_node_id: the node label
self.actual_cims = [] :_parents_states_number: the cardinalities of the parents
self.state_residence_times = None :_node_states_number: the caridinality of the node
self.transition_matrices = None :_p_combs: the relative p_comb structure
self.p_combs = p_combs
"""
self._node_id = node_id
self._parents_states_number = parents_states_number
self._node_states_number = node_states_number
self._actual_cims = []
self._state_residence_times = None
self._transition_matrices = None
self._p_combs = p_combs
self.build_times_and_transitions_structures() self.build_times_and_transitions_structures()
def build_times_and_transitions_structures(self): def build_times_and_transitions_structures(self):
@ -34,73 +41,64 @@ class SetOfCims:
Initializes at the correct dimensions the state residence times matrix and the state transition matrices Initializes at the correct dimensions the state residence times matrix and the state transition matrices
Parameters: Parameters:
void :void
Returns: Returns:
void :void
""" """
if not self.parents_states_number: if not self._parents_states_number:
self.state_residence_times = np.zeros((1, self.node_states_number), dtype=np.float) self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float)
self.transition_matrices = np.zeros((1,self.node_states_number, self.node_states_number), dtype=np.int) self._transition_matrices = np.zeros((1, self._node_states_number, self._node_states_number), dtype=np.int)
else: else:
self.state_residence_times = \ self._state_residence_times = \
np.zeros((np.prod(self.parents_states_number), self.node_states_number), dtype=np.float) np.zeros((np.prod(self._parents_states_number), self._node_states_number), dtype=np.float)
self.transition_matrices = np.zeros([np.prod(self.parents_states_number), self.node_states_number, self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number,
self.node_states_number], dtype=np.int) self._node_states_number], dtype=np.int)
def build_cims(self, state_res_times: typing.List, transition_matrices: typing.List): def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray):
""" """
Build the ConditionalIntensityMatrix object given the state residence times and transitions matrices. Build the ConditionalIntensityMatrix object given the state residence times and transitions matrices.
Compute the cim coefficients. Compute the cim coefficients.
Parameters: Parameters:
state_res_times: the state residence times matrix :state_res_times: the state residence times matrix
transition_matrices: the transition matrices :_transition_matrices: the transition matrices
Returns: Returns:
void :void
""" """
for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices): for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices):
cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix) cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix)
cim_to_add.compute_cim_coefficients() cim_to_add.compute_cim_coefficients()
self.actual_cims.append(cim_to_add) self._actual_cims.append(cim_to_add)
self.actual_cims = np.array(self.actual_cims) self._actual_cims = np.array(self._actual_cims)
self.transition_matrices = None self._transition_matrices = None
self.state_residence_times = None self._state_residence_times = None
def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray: def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray:
""" """
Filter the cims contained in the array actual_cims given the boolean mask mask_arr and the index comb. Filter the cims contained in the array _actual_cims given the boolean mask mask_arr and the index comb.
Parameters: Parameters:
mask_arr: the boolean mask :mask_arr: the boolean mask
comb: the indexes of the selected cims :comb: the indexes of the selected cims
Returns: Returns:
Array of ConditionalIntensityMatrix :Array of ConditionalIntensityMatrix
""" """
if mask_arr.size <= 1: if mask_arr.size <= 1:
return self.actual_cims return self._actual_cims
else: else:
flat_indxs = np.argwhere(np.all(self.p_combs[:, mask_arr] == comb, axis=1)).ravel() flat_indxs = np.argwhere(np.all(self._p_combs[:, mask_arr] == comb, axis=1)).ravel()
return self.actual_cims[flat_indxs] return self._actual_cims[flat_indxs]
@property @property
def get_cims(self): def actual_cims(self) -> np.ndarray:
return self.actual_cims return self._actual_cims
@property
def p_combs(self) -> np.ndarray:
return self._p_combs
def get_cims_number(self): def get_cims_number(self):
return len(self.actual_cims) return len(self._actual_cims)
"""
def get_cim(self, index):
flat_index = self.indexes_converter(index)
return self.actual_cims[flat_index]
def indexes_converter(self, indexes):
assert len(indexes) == len(self.parents_states_number)
vector_index = 0
if not indexes:
return vector_index
else:
for indx, value in enumerate(indexes):
vector_index = vector_index*self.parents_states_number[indx] + indexes[indx]
return vector_index"""

2
main_package/classes/simple_cvs_importer.py
Unescape View File

@ -19,7 +19,7 @@ class CSVImporter(ai.AbstractImporter):
self.compute_row_delta_in_all_samples_frames(self._df_samples_list) self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
def read_csv_file(self): def read_csv_file(self):
df = pd.read_csv(self.file_path) df = pd.read_csv(self._file_path)
df.drop(df.columns[[0]], axis=1, inplace=True) df.drop(df.columns[[0]], axis=1, inplace=True)
self._df_samples_list = [df] self._df_samples_list = [df]

17
main_package/classes/structure.py
Unescape View File

@ -16,6 +16,15 @@ class Structure:
def __init__(self, nodes_label_list: ty.List, node_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray, def __init__(self, nodes_label_list: ty.List, node_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
edges_list: ty.List, total_variables_number: int): edges_list: ty.List, total_variables_number: int):
"""
Parameters:
:nodes_labels_list: the symbolic names of the variables
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
self._nodes_labels_list = nodes_label_list self._nodes_labels_list = nodes_label_list
self._nodes_indexes_arr = node_indexes_arr self._nodes_indexes_arr = node_indexes_arr
self._nodes_vals_arr = nodes_vals_arr self._nodes_vals_arr = nodes_vals_arr
@ -23,13 +32,11 @@ class Structure:
self._total_variables_number = total_variables_number self._total_variables_number = total_variables_number
@property @property
def edges(self): def edges(self) -> ty.List:
#records = self.structure_frame.to_records(index=False)
#edges_list = list(records)
return self._edges_list return self._edges_list
@property @property
def nodes_labels(self): def nodes_labels(self) -> ty.List:
return self._nodes_labels_list return self._nodes_labels_list
@property @property
@ -41,7 +48,7 @@ class Structure:
return self._nodes_vals_arr return self._nodes_vals_arr
@property @property
def total_variables_number(self): def total_variables_number(self) -> int:
return self._total_variables_number return self._total_variables_number
def get_node_id(self, node_indx: int) -> str: def get_node_id(self, node_indx: int) -> str:

183
main_package/classes/structure_estimator.py
Unescape View File

@ -21,28 +21,35 @@ class StructureEstimator:
""" """
Has the task of estimating the network structure given the trajectories in samplepath. Has the task of estimating the network structure given the trajectories in samplepath.
:sample_path: the sample_path object containing the trajectories and the real structure :_sample_path: the _sample_path object containing the trajectories and the real structure
:exp_test_sign: the significance level for the exponential Hp test :_exp_test_sign: the significance level for the exponential Hp test
:chi_test_alfa: the significance level for the chi Hp test :_chi_test_alfa: the significance level for the chi Hp test
:nodes: the nodes labels :_nodes: the _nodes labels
:nodes_vals: the nodes cardinalities :_nodes_vals: the nodes cardinalities
:nodes_indxs: the nodes indexes :_nodes_indxs: the nodes indexes
:complete_graph: the complete directed graph built using the nodes labels in nodes :_complete_graph: the complete directed graph built using the nodes labels in nodes
:cache: the cache object :_cache: the _cache object
""" """
def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float): def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float):
self.sample_path = sample_path """
self.nodes = np.array(self.sample_path.structure.nodes_labels) Parameters:
self.nodes_vals = self.sample_path.structure.nodes_values :_sample_path: the _sample_path object containing the trajectories and the real structure
self.nodes_indxs = self.sample_path.structure.nodes_indexes :_exp_test_sign: the significance level for the exponential Hp test
self.complete_graph = self.build_complete_graph(self.sample_path.structure.nodes_labels) :_chi_test_alfa: the significance level for the chi Hp test
self.exp_test_sign = exp_test_alfa
self.chi_test_alfa = chi_test_alfa """
self.cache = ch.Cache() self._sample_path = sample_path
self._nodes = np.array(self._sample_path.structure.nodes_labels)
self._nodes_vals = self._sample_path.structure.nodes_values
self._nodes_indxs = self._sample_path.structure.nodes_indexes
self._complete_graph = self.build_complete_graph(self._sample_path.structure.nodes_labels)
self._exp_test_sign = exp_test_alfa
self._chi_test_alfa = chi_test_alfa
self._cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List): def build_complete_graph(self, node_ids: typing.List) -> nx.DiGraph:
""" """
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids: Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
@ -57,9 +64,9 @@ class StructureEstimator:
return complete_graph return complete_graph
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int, def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int): tot_vars_count: int) -> bool:
""" """
Permorms a complete independence test on the directed graphs G1 = test_child U parent_set Performs a complete independence test on the directed graphs G1 = test_child U parent_set
G2 = G1 U test_parent (added as an additional parent of the test_child). G2 = G1 U test_parent (added as an additional parent of the test_child).
Generates all the necessary structures and datas to perform the tests. Generates all the necessary structures and datas to perform the tests.
@ -68,98 +75,61 @@ class StructureEstimator:
test_child: the node label of the child test_child: the node label of the child
parent_set: the common parent set parent_set: the common parent set
child_states_numb: the cardinality of the test_child child_states_numb: the cardinality of the test_child
tot_vars_count_ the total number of variables in the net tot_vars_count: the total number of variables in the net
Returns: Returns:
True iff test_child and test_parent are independent given the sep_set parent_set True iff test_child and test_parent are independent given the sep_set parent_set
False otherwise False otherwise
""" """
#print("Test Parent:", test_parent)
#print("Sep Set", parent_set)
p_set = parent_set[:] p_set = parent_set[:]
complete_info = parent_set[:] complete_info = parent_set[:]
complete_info.append(test_child) complete_info.append(test_child)
parents = np.array(parent_set) parents = np.array(parent_set)
parents = np.append(parents, test_parent) parents = np.append(parents, test_parent)
#print("PARENTS", parents) sorted_parents = self._nodes[np.isin(self._nodes, parents)]
#parents.sort()
sorted_parents = self.nodes[np.isin(self.nodes, parents)]
#print("SORTED PARENTS", sorted_parents)
cims_filter = sorted_parents != test_parent cims_filter = sorted_parents != test_parent
#print("PARENTS NO FROM MASK", cims_filter) sofc1 = self._cache.find(set(p_set))
#if not p_set:
#print("EMPTY PSET TRYING TO FIND", test_child)
#sofc1 = self.cache.find(test_child)
#else:
sofc1 = self.cache.find(set(p_set))
if not sofc1: if not sofc1:
#print("CACHE MISSS SOFC1") bool_mask1 = np.isin(self._nodes, complete_info)
bool_mask1 = np.isin(self.nodes,complete_info) l1 = list(self._nodes[bool_mask1])
#print("Bool mask 1", bool_mask1) indxs1 = self._nodes_indxs[bool_mask1]
l1 = list(self.nodes[bool_mask1]) vals1 = self._nodes_vals[bool_mask1]
#print("L1", l1)
indxs1 = self.nodes_indxs[bool_mask1]
#print("INDXS 1", indxs1)
vals1 = self.nodes_vals[bool_mask1]
eds1 = list(itertools.product(parent_set,test_child)) eds1 = list(itertools.product(parent_set,test_child))
s1 = st.Structure(l1, indxs1, vals1, eds1, tot_vars_count) s1 = st.Structure(l1, indxs1, vals1, eds1, tot_vars_count)
g1 = ng.NetworkGraph(s1) g1 = ng.NetworkGraph(s1)
g1.fast_init(test_child) g1.fast_init(test_child)
p1 = pe.ParametersEstimator(self.sample_path, g1) p1 = pe.ParametersEstimator(self._sample_path.trajectories, g1)
p1.fast_init(test_child) p1.fast_init(test_child)
sofc1 = p1.compute_parameters_for_node(test_child) sofc1 = p1.compute_parameters_for_node(test_child)
#if not p_set: self._cache.put(set(p_set), sofc1)
#self.cache.put(test_child, sofc1)
#else:
self.cache.put(set(p_set), sofc1)
sofc2 = None sofc2 = None
#p_set.append(test_parent)
p_set.insert(0, test_parent) p_set.insert(0, test_parent)
if p_set: if p_set:
#print("FULL PSET TRYING TO FIND", p_set) sofc2 = self._cache.find(set(p_set))
#p_set.append(test_parent)
#print("PSET ", p_set)
#set_p_set = set(p_set)
sofc2 = self.cache.find(set(p_set))
#if sofc2:
#print("Sofc2 in CACHE ", sofc2.actual_cims)
#print(self.cache.list_of_sets_of_indxs)
if not sofc2: if not sofc2:
#print("Cache MISSS SOFC2")
complete_info.append(test_parent) complete_info.append(test_parent)
bool_mask2 = np.isin(self.nodes, complete_info) bool_mask2 = np.isin(self._nodes, complete_info)
#print("BOOL MASK 2",bool_mask2) l2 = list(self._nodes[bool_mask2])
l2 = list(self.nodes[bool_mask2]) indxs2 = self._nodes_indxs[bool_mask2]
#print("L2", l2) vals2 = self._nodes_vals[bool_mask2]
indxs2 = self.nodes_indxs[bool_mask2]
#print("INDXS 2", indxs2)
vals2 = self.nodes_vals[bool_mask2]
eds2 = list(itertools.product(p_set, test_child)) eds2 = list(itertools.product(p_set, test_child))
s2 = st.Structure(l2, indxs2, vals2, eds2, tot_vars_count) s2 = st.Structure(l2, indxs2, vals2, eds2, tot_vars_count)
g2 = ng.NetworkGraph(s2) g2 = ng.NetworkGraph(s2)
g2.fast_init(test_child) g2.fast_init(test_child)
p2 = pe.ParametersEstimator(self.sample_path, g2) p2 = pe.ParametersEstimator(self._sample_path.trajectories, g2)
p2.fast_init(test_child) p2.fast_init(test_child)
sofc2 = p2.compute_parameters_for_node(test_child) sofc2 = p2.compute_parameters_for_node(test_child)
self.cache.put(set(p_set), sofc2) self._cache.put(set(p_set), sofc2)
for cim1, p_comb in zip(sofc1.actual_cims, sofc1.p_combs): for cim1, p_comb in zip(sofc1.actual_cims, sofc1.p_combs):
#print("GETTING THIS P COMB", p_comb)
#if len(parent_set) > 1:
cond_cims = sofc2.filter_cims_with_mask(cims_filter, p_comb) cond_cims = sofc2.filter_cims_with_mask(cims_filter, p_comb)
#else:
#cond_cims = sofc2.actual_cims
#print("COnd Cims", cond_cims)
for cim2 in cond_cims: for cim2 in cond_cims:
#cim2 = sofc2.actual_cims[j]
#print(indx)
#print("Run Test", i, j)
if not self.independence_test(child_states_numb, cim1, cim2): if not self.independence_test(child_states_numb, cim1, cim2):
return False return False
return True return True
def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix, def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix,
cim2: condim.ConditionalIntensityMatrix): cim2: condim.ConditionalIntensityMatrix) -> bool:
""" """
Compute the actual independence test using two cims. Compute the actual independence test using two cims.
It is performed first the exponential test and if the null hypothesis is not rejected, It is performed first the exponential test and if the null hypothesis is not rejected,
@ -181,38 +151,23 @@ class StructureEstimator:
C1 = cim1.cim C1 = cim1.cim
C2 = cim2.cim C2 = cim2.cim
F_stats = C2.diagonal() / C1.diagonal() F_stats = C2.diagonal() / C1.diagonal()
exp_alfa = self.exp_test_sign exp_alfa = self._exp_test_sign
for val in range(0, child_states_numb): for val in range(0, child_states_numb):
if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \ if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \
F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]): F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]):
#print("CONDITIONALLY DEPENDENT EXP") #print("CONDITIONALLY DEPENDENT EXP")
return False return False
#M1_no_diag = self.remove_diagonal_elements(cim1.state_transition_matrix)
#M2_no_diag = self.remove_diagonal_elements(cim2.state_transition_matrix)
M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1) M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1)
M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape( M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape(
M2.shape[0], -1) M2.shape[0], -1)
chi_2_quantile = chi2_dist.ppf(1 - self.chi_test_alfa, child_states_numb - 1) chi_2_quantile = chi2_dist.ppf(1 - self._chi_test_alfa, child_states_numb - 1)
"""
Ks = np.sqrt(cim1.state_transition_matrix.diagonal() / cim2.state_transition_matrix.diagonal())
Ls = np.reciprocal(Ks)
chi_stats = np.sum((np.power((M2_no_diag.T * Ks).T - (M1_no_diag.T * Ls).T, 2) \
/ (M1_no_diag + M2_no_diag)), axis=1)"""
Ks = np.sqrt(r1s / r2s) Ks = np.sqrt(r1s / r2s)
Ls = np.sqrt(r2s / r1s) Ls = np.sqrt(r2s / r1s)
for val in range(0, child_states_numb): for val in range(0, child_states_numb):
#K = math.sqrt(cim1.state_transition_matrix[val][val] / cim2.state_transition_matrix[val][val])
#L = 1 / K
Chi = np.sum(np.power(Ks[val] * M2_no_diag[val] - Ls[val] *M1_no_diag[val], 2) / Chi = np.sum(np.power(Ks[val] * M2_no_diag[val] - Ls[val] *M1_no_diag[val], 2) /
(M1_no_diag[val] + M2_no_diag[val])) (M1_no_diag[val] + M2_no_diag[val]))
#print("Chi Stats", Chi)
#print("Chi Quantile", chi_2_quantile)
if Chi > chi_2_quantile: if Chi > chi_2_quantile:
#if np.any(chi_stats > chi_2_quantile):
#print("CONDITIONALLY DEPENDENT CHI")
return False return False
#print("Chi test", Chi)
return True return True
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int): def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int):
@ -221,54 +176,45 @@ class StructureEstimator:
Parameters: Parameters:
var_id: the node label of the test child var_id: the node label of the test child
tot_vars_count: the number of nodes in the net tot_vars_count: the number of _nodes in the net
Returns: Returns:
void void
""" """
print("##################TESTING VAR################", var_id) print("##################TESTING VAR################", var_id)
u = list(self.complete_graph.predecessors(var_id)) u = list(self._complete_graph.predecessors(var_id))
#tests_parents_numb = len(u) child_states_numb = self._sample_path.structure.get_states_number(var_id)
#complete_frame = self.complete_graph_frame
#test_frame = complete_frame.loc[complete_frame['To'].isin([var_id])]
child_states_numb = self.sample_path.structure.get_states_number(var_id)
b = 0 b = 0
while b < len(u): while b < len(u):
#for parent_id in u:
parent_indx = 0 parent_indx = 0
while parent_indx < len(u): while parent_indx < len(u):
removed = False removed = False
#if not list(self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])):
#break
S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx]) S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])
#print("U Set", u)
#print("S", S)
test_parent = u[parent_indx] test_parent = u[parent_indx]
#print("Test Parent", test_parent) #print("Test Parent", test_parent)
for parents_set in S: for parents_set in S:
#print("Parent Set", parents_set) print("Parent Set", parents_set)
#print("Test Parent", test_parent) print("Test Parent", test_parent)
if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count): if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count):
#print("Removing EDGE:", test_parent, var_id) #print("Removing EDGE:", test_parent, var_id)
self.complete_graph.remove_edge(test_parent, var_id) self._complete_graph.remove_edge(test_parent, var_id)
u.remove(test_parent) u.remove(test_parent)
removed = True removed = True
break break
#else:
#parent_indx += 1
if not removed: if not removed:
parent_indx += 1 parent_indx += 1
b += 1 b += 1
self.cache.clear() self._cache.clear()
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str): def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str) -> \
typing.Iterator:
""" """
Creates a list containing all possible subsets of the list u of size size, Creates a list containing all possible subsets of the list u of size size,
that do not contains a the node identified by parent_label. that do not contains a the node identified by parent_label.
Parameters: Parameters:
u: the list of nodes u: the list of _nodes
size: the size of the subsets size: the size of the subsets
parent_label: the nodes to exclude in the subsets generation parent_label: the _nodes to exclude in the subsets generation
Returns: Returns:
a Map Object containing a list of lists a Map Object containing a list of lists
@ -286,8 +232,8 @@ class StructureEstimator:
void void
""" """
ctpc_algo = self.one_iteration_of_CTPC_algorithm ctpc_algo = self.one_iteration_of_CTPC_algorithm
total_vars_numb = self.sample_path.total_variables_count total_vars_numb = self._sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self.nodes] [ctpc_algo(n, total_vars_numb) for n in self._nodes]
def save_results(self): def save_results(self):
""" """
@ -298,17 +244,10 @@ class StructureEstimator:
Returns: Returns:
void void
""" """
res = json_graph.node_link_data(self.complete_graph) res = json_graph.node_link_data(self._complete_graph)
name = self.sample_path.importer.file_path.rsplit('/',1)[-1] name = self._sample_path._importer.file_path.rsplit('/', 1)[-1]
#print(name) name = 'results_' + name #TODO va aggiunto anche l'indice di array
name = 'results_' + name
with open(name, 'w') as f: with open(name, 'w') as f:
json.dump(res, f) json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)

10
main_package/classes/trajectory.py
Unescape View File

@ -15,9 +15,15 @@ class Trajectory:
""" """
def __init__(self, list_of_columns, original_cols_number): def __init__(self, list_of_columns, original_cols_number):
"""
Parameters:
:list_of_columns: the list containing the times array and values matrix
:original_cols_numb: total number of cols in the data
"""
if type(list_of_columns[0][0]) != np.float64: if type(list_of_columns[0][0]) != np.float64:
raise TypeError('The first array in the list has to be Times') raise TypeError('The first array in the list has to be Times')
self.original_cols_number = original_cols_number self._original_cols_number = original_cols_number
self._actual_trajectory = np.array(list_of_columns[1:], dtype=np.int).T self._actual_trajectory = np.array(list_of_columns[1:], dtype=np.int).T
self._times = np.array(list_of_columns[0], dtype=np.float) self._times = np.array(list_of_columns[0], dtype=np.float)
@ -29,7 +35,7 @@ class Trajectory:
Returns: Returns:
a numpy matrix containing ONLY the original columns values, not the shifted ones a numpy matrix containing ONLY the original columns values, not the shifted ones
""" """
return self._actual_trajectory[:, :self.original_cols_number] return self._actual_trajectory[:, :self._original_cols_number]
@property @property
def complete_trajectory(self) -> np.ndarray: def complete_trajectory(self) -> np.ndarray:

28
main_package/tests/test_cache.py
Unescape View File

@ -11,31 +11,31 @@ class TestCache(unittest.TestCase):
def test_init(self): def test_init(self):
c1 = ch.Cache() c1 = ch.Cache()
self.assertFalse(c1.list_of_sets_of_parents) self.assertFalse(c1._list_of_sets_of_parents)
self.assertFalse(c1.actual_cache) self.assertFalse(c1._actual_cache)
def test_put(self): def test_put(self):
c1 = ch.Cache() c1 = ch.Cache()
pset1 = {'X', 'Y'} pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([])) sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1) c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache)) self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents)) self.assertEqual(1, len(c1._list_of_sets_of_parents))
self.assertEqual(sofc1, c1.actual_cache[0]) self.assertEqual(sofc1, c1._actual_cache[0])
pset2 = {'X'} pset2 = {'X'}
sofc2 = soci.SetOfCims('Z', [], 3, np.array([])) sofc2 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset2, sofc2) c1.put(pset2, sofc2)
self.assertEqual(2, len(c1.actual_cache)) self.assertEqual(2, len(c1._actual_cache))
self.assertEqual(2, len(c1.list_of_sets_of_parents)) self.assertEqual(2, len(c1._list_of_sets_of_parents))
self.assertEqual(sofc2, c1.actual_cache[1]) self.assertEqual(sofc2, c1._actual_cache[1])
def test_find(self): def test_find(self):
c1 = ch.Cache() c1 = ch.Cache()
pset1 = {'X', 'Y'} pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([])) sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1) c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache)) self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents)) self.assertEqual(1, len(c1._list_of_sets_of_parents))
self.assertIsInstance(c1.find(pset1), soci.SetOfCims) self.assertIsInstance(c1.find(pset1), soci.SetOfCims)
self.assertEqual(sofc1, c1.find(pset1)) self.assertEqual(sofc1, c1.find(pset1))
self.assertIsInstance(c1.find({'Y', 'X'}), soci.SetOfCims) self.assertIsInstance(c1.find({'Y', 'X'}), soci.SetOfCims)
@ -47,11 +47,11 @@ class TestCache(unittest.TestCase):
pset1 = {'X', 'Y'} pset1 = {'X', 'Y'}
sofc1 = soci.SetOfCims('Z', [], 3, np.array([])) sofc1 = soci.SetOfCims('Z', [], 3, np.array([]))
c1.put(pset1, sofc1) c1.put(pset1, sofc1)
self.assertEqual(1, len(c1.actual_cache)) self.assertEqual(1, len(c1._actual_cache))
self.assertEqual(1, len(c1.list_of_sets_of_parents)) self.assertEqual(1, len(c1._list_of_sets_of_parents))
c1.clear() c1.clear()
self.assertFalse(c1.list_of_sets_of_parents) self.assertFalse(c1._list_of_sets_of_parents)
self.assertFalse(c1.actual_cache) self.assertFalse(c1._actual_cache)

96
main_package/tests/test_json_importer.py
Unescape View File

@ -16,14 +16,14 @@ class TestJsonImporter(unittest.TestCase):
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
def test_init(self): def test_init(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
self.assertEqual(j1.samples_label, 'samples') self.assertEqual(j1._samples_label, 'samples')
self.assertEqual(j1.structure_label, 'dyn.str') self.assertEqual(j1._structure_label, 'dyn.str')
self.assertEqual(j1.variables_label, 'variables') self.assertEqual(j1._variables_label, 'variables')
self.assertEqual(j1.time_key, 'Time') self.assertEqual(j1._time_key, 'Time')
self.assertEqual(j1.variables_key, 'Name') self.assertEqual(j1._variables_key, 'Name')
self.assertEqual(j1.file_path, self.read_files[0]) self.assertEqual(j1._file_path, self.read_files[0])
self.assertIsNone(j1.df_samples_list) self.assertIsNone(j1._df_samples_list)
self.assertIsNone(j1.variables) self.assertIsNone(j1.variables)
self.assertIsNone(j1.structure) self.assertIsNone(j1.structure)
self.assertIsNone(j1.concatenated_samples) self.assertIsNone(j1.concatenated_samples)
@ -35,7 +35,7 @@ class TestJsonImporter(unittest.TestCase):
json.dump(data_set, f) json.dump(data_set, f)
path = os.getcwd() path = os.getcwd()
path = path + '/data.json' path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '') j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
imported_data = j1.read_json_file() imported_data = j1.read_json_file()
self.assertTrue(self.ordered(data_set) == self.ordered(imported_data)) self.assertTrue(self.ordered(data_set) == self.ordered(imported_data))
os.remove('data.json') os.remove('data.json')
@ -43,32 +43,32 @@ class TestJsonImporter(unittest.TestCase):
def test_read_json_file_not_found(self): def test_read_json_file_not_found(self):
path = os.getcwd() path = os.getcwd()
path = path + '/data.json' path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '') j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
self.assertRaises(FileNotFoundError, j1.read_json_file) self.assertRaises(FileNotFoundError, j1.read_json_file)
def test_normalize_trajectories(self): def test_normalize_trajectories(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
#print(raw_data) #print(raw_data)
df_samples_list = j1.normalize_trajectories(raw_data, 0, j1.samples_label) df_samples_list = j1.normalize_trajectories(raw_data, 0, j1._samples_label)
self.assertEqual(len(df_samples_list), len(raw_data[0][j1.samples_label])) self.assertEqual(len(df_samples_list), len(raw_data[0][j1._samples_label]))
#self.assertEqual(list(j1.df_samples_list[0].columns.values)[1:], j1.sorter) #self.assertEqual(list(j1._df_samples_list[0].columns.values)[1:], j1.sorter)
def test_normalize_trajectories_wrong_indx(self): def test_normalize_trajectories_wrong_indx(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
self.assertRaises(IndexError, j1.normalize_trajectories, raw_data, 474, j1.samples_label) self.assertRaises(IndexError, j1.normalize_trajectories, raw_data, 474, j1._samples_label)
def test_normalize_trajectories_wrong_key(self): def test_normalize_trajectories_wrong_key(self):
j1 = ji.JsonImporter(self.read_files[0], 'sample', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'sample', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
self.assertRaises(KeyError, j1.normalize_trajectories, raw_data, 0, j1.samples_label) self.assertRaises(KeyError, j1.normalize_trajectories, raw_data, 0, j1._samples_label)
def test_compute_row_delta_single_samples_frame(self): def test_compute_row_delta_single_samples_frame(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data) j1._df_samples_list = j1.import_trajectories(raw_data)
sample_frame = j1.df_samples_list[0] sample_frame = j1._df_samples_list[0]
original_copy = sample_frame.copy() original_copy = sample_frame.copy()
columns_header = list(sample_frame.columns.values) columns_header = list(sample_frame.columns.values)
shifted_cols_header = [s + "S" for s in columns_header[1:]] shifted_cols_header = [s + "S" for s in columns_header[1:]]
@ -87,27 +87,27 @@ class TestJsonImporter(unittest.TestCase):
np.array(original_copy.iloc[indx + 1][columns_header[1:]], dtype=int)) np.array(original_copy.iloc[indx + 1][columns_header[1:]], dtype=int))
def test_compute_row_delta_in_all_frames(self): def test_compute_row_delta_in_all_frames(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data) j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0]) j1._sorter = j1.build_sorter(j1._df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1.df_samples_list) j1.compute_row_delta_in_all_samples_frames(j1._df_samples_list)
self.assertEqual(list(j1.df_samples_list[0].columns.values), self.assertEqual(list(j1._df_samples_list[0].columns.values),
list(j1.concatenated_samples.columns.values)[:len(list(j1.df_samples_list[0].columns.values))]) list(j1.concatenated_samples.columns.values)[:len(list(j1._df_samples_list[0].columns.values))])
self.assertEqual(list(j1.concatenated_samples.columns.values)[0], j1.time_key) self.assertEqual(list(j1.concatenated_samples.columns.values)[0], j1._time_key)
def test_clear_data_frame_list(self): def test_clear_data_frame_list(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data) j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0]) j1._sorter = j1.build_sorter(j1._df_samples_list[0])
j1.compute_row_delta_in_all_samples_frames(j1.df_samples_list) j1.compute_row_delta_in_all_samples_frames(j1._df_samples_list)
j1.clear_data_frame_list() j1.clear_data_frame_list()
for df in j1.df_samples_list: for df in j1._df_samples_list:
self.assertTrue(df.empty) self.assertTrue(df.empty)
def test_clear_concatenated_frame(self): def test_clear_concatenated_frame(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
j1.import_data() j1.import_data()
j1.clear_concatenated_frame() j1.clear_concatenated_frame()
self.assertTrue(j1.concatenated_samples.empty) self.assertTrue(j1.concatenated_samples.empty)
@ -118,7 +118,7 @@ class TestJsonImporter(unittest.TestCase):
json.dump(data_set, f) json.dump(data_set, f)
path = os.getcwd() path = os.getcwd()
path = path + '/data.json' path = path + '/data.json'
j1 = ji.JsonImporter(path, '', '', '', '', '') j1 = ji.JsonImporter(path, '', '', '', '', '', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
frame = pd.DataFrame(raw_data) frame = pd.DataFrame(raw_data)
col_list = j1.build_list_of_samples_array(frame) col_list = j1.build_list_of_samples_array(frame)
@ -130,39 +130,33 @@ class TestJsonImporter(unittest.TestCase):
os.remove('data.json') os.remove('data.json')
def test_import_variables(self): def test_import_variables(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
sorter = ['X', 'Y', 'Z'] sorter = ['X', 'Y', 'Z']
raw_data = [{'variables':{"Name": ['X', 'Y', 'Z'], "value": [3, 3, 3]}}] raw_data = [{'variables':{"Name": ['X', 'Y', 'Z'], "value": [3, 3, 3]}}]
df_var = j1.import_variables(raw_data, sorter) df_var = j1.import_variables(raw_data)
self.assertEqual(list(df_var[j1.variables_key]), sorter) self.assertEqual(list(df_var[j1._variables_key]), sorter)
def test_import_structure(self): def test_import_structure(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = [{"dyn.str":[{"From":"X","To":"Z"},{"From":"Y","To":"Z"},{"From":"Z","To":"Y"}]}] raw_data = [{"dyn.str":[{"From":"X","To":"Z"},{"From":"Y","To":"Z"},{"From":"Z","To":"Y"}]}]
df_struct = j1.import_structure(raw_data) df_struct = j1.import_structure(raw_data)
#print(raw_data[0]['dyn.str'][0].items()) #print(raw_data[0]['dyn.str'][0].items())
self.assertIsInstance(df_struct, pd.DataFrame) self.assertIsInstance(df_struct, pd.DataFrame)
def test_import_sampled_cims(self): def test_import_sampled_cims(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
raw_data = j1.read_json_file() raw_data = j1.read_json_file()
j1.df_samples_list = j1.import_trajectories(raw_data) j1._df_samples_list = j1.import_trajectories(raw_data)
j1._sorter = j1.build_sorter(j1.df_samples_list[0]) j1._sorter = j1.build_sorter(j1._df_samples_list[0])
cims = j1.import_sampled_cims(raw_data, 0, 'dyn.cims') cims = j1.import_sampled_cims(raw_data, 0, 'dyn.cims')
#j1.import_variables(raw_data, j1.sorter) #j1.import_variables(raw_data, j1.sorter)
self.assertEqual(list(cims.keys()), j1.sorter) self.assertEqual(list(cims.keys()), j1.sorter)
def test_import_data(self): def test_import_data(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 2)
j1.import_data() j1.import_data()
#lp = LineProfiler() self.assertEqual(list(j1.variables[j1._variables_key]),
list(j1.concatenated_samples.columns.values[1:len(j1.variables[j1._variables_key]) + 1]))
#lp_wrapper = lp(j1.import_data)
#lp_wrapper()
#lp.print_stats()
#j1.import_data()
self.assertEqual(list(j1.variables[j1.variables_key]),
list(j1.concatenated_samples.columns.values[1:len(j1.variables[j1.variables_key]) + 1]))
print(j1.variables) print(j1.variables)
print(j1.structure) print(j1.structure)
print(j1.concatenated_samples) print(j1.concatenated_samples)

60
main_package/tests/test_networkgraph.py
Unescape View File

@ -12,20 +12,19 @@ import network_graph as ng
import json_importer as ji import json_importer as ji
class TestNetworkGraph(unittest.TestCase): class TestNetworkGraph(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
cls.s1 = sp.SamplePath(cls.importer) cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories() cls.s1.build_trajectories()
cls.s1.build_structure() cls.s1.build_structure()
def test_init(self): def test_init(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
self.assertEqual(self.s1.structure, g1.graph_struct) self.assertEqual(self.s1.structure, g1._graph_struct)
self.assertIsInstance(g1.graph, nx.DiGraph) self.assertIsInstance(g1._graph, nx.DiGraph)
self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes)) self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes))
self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels) self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels)
self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values)) self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values))
@ -59,7 +58,7 @@ class TestNetworkGraph(unittest.TestCase):
self.assertEqual(graph.get_node_indx(node), node_indx) self.assertEqual(graph.get_node_indx(node), node_indx)
else: else:
for node, node_val in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[2]): for node, node_val in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[2]):
self.assertEqual(graph.graph_struct.get_states_number(node), node_val) self.assertEqual(graph._graph_struct.get_states_number(node), node_val)
def test_get_ord_set_of_par_of_all_nodes(self): def test_get_ord_set_of_par_of_all_nodes(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
@ -68,36 +67,7 @@ class TestNetworkGraph(unittest.TestCase):
sorted_list_of_par_lists = g1.get_ord_set_of_par_of_all_nodes() sorted_list_of_par_lists = g1.get_ord_set_of_par_of_all_nodes()
for node, par_list in zip(g1.nodes, sorted_list_of_par_lists): for node, par_list in zip(g1.nodes, sorted_list_of_par_lists):
self.aux_aggregated_par_list_data(g1, node, par_list) self.aux_aggregated_par_list_data(g1, node, par_list)
"""
def test_get_ordered_by_indx_parents_values_for_all_nodes(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
#print(g1.get_ordered_by_indx_parents_values_for_all_nodes())
parents_values_list = g1.get_ordered_by_indx_parents_values_for_all_nodes()
for pv1, aggr in zip(parents_values_list, g1.aggregated_info_about_nodes_parents):
self.assertEqual(pv1, aggr[2])
def test_get_states_number_of_all_nodes_sorted(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
nodes_cardinality_list = g1.get_states_number_of_all_nodes_sorted()
for val, node in zip(nodes_cardinality_list, g1.get_nodes_sorted_by_indx()):
self.assertEqual(val, g1.get_states_number(node))
def test_build_fancy_indexing_structure_no_offset(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.list_of_nodes_labels())
g1.add_edges(self.s1.structure.list_of_edges())
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
for par_indxs, aggr in zip(fancy_indx, g1.aggregated_info_about_nodes_parents):
self.assertEqual(par_indxs, aggr[1])
def test_build_fancy_indexing_structure_offset(self):
pass #TODO il codice di netgraph deve gestire questo caso"""
def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals): def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals):
time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals) time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals)
@ -130,30 +100,30 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_labels = [i[0] for i in g1.aggregated_info_about_nodes_parents] p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes() p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0) fancy_indx = g1.build_fancy_indexing_structure(0)
for node_id, p_i ,p_l, p_v in zip(g1.graph_struct.nodes_labels, fancy_indx, p_labels, p_vals): for node_id, p_i ,p_l, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node_id, p_i ,p_l, p_v) self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node_id, p_i ,p_l, p_v)
def test_build_time_scalar_indexing_structure(self): def test_build_time_scalar_indexing_structure(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0) fancy_indx = g1.build_fancy_indexing_structure(0)
p_labels = [i[0] for i in g1.aggregated_info_about_nodes_parents] p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes() p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
#print(fancy_indx) #print(fancy_indx)
for node_id, p_indxs, p_labels, p_v in zip(g1.graph_struct.nodes_labels, fancy_indx, p_labels, p_vals): for node_id, p_indxs, p_labels, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node_id, p_indxs, p_labels, p_v) self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node_id, p_indxs, p_labels, p_v)
def test_build_time_columns_filtering_structure(self): def test_build_time_columns_filtering_structure(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0) g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_time_columns_filtering_structure() g1.build_time_columns_filtering_structure()
t_filter = [] t_filter = []
@ -170,13 +140,13 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0) g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_transition_columns_filtering_structure() g1.build_transition_columns_filtering_structure()
m_filter = [] m_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing): for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing):
single_filter = [] single_filter = []
single_filter.append(g1.get_node_indx(node_id) + g1.graph_struct.total_variables_number) single_filter.append(g1.get_node_indx(node_id) + g1._graph_struct.total_variables_number)
single_filter.append(g1.get_node_indx(node_id)) single_filter.append(g1.get_node_indx(node_id))
single_filter.extend(p_indxs) single_filter.extend(p_indxs)
m_filter.append(np.array(single_filter)) m_filter.append(np.array(single_filter))
@ -187,7 +157,7 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1.aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes() p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs = g1.build_p_combs_structure() p_combs = g1.build_p_combs_structure()
@ -219,7 +189,7 @@ class TestNetworkGraph(unittest.TestCase):
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
for node in g1.nodes: for node in g1.nodes:
self.assertListEqual(g1.get_parents_by_id(node), list(g1.graph.predecessors(node))) self.assertListEqual(g1.get_parents_by_id(node), list(g1._graph.predecessors(node)))
def test_get_states_number(self): def test_get_states_number(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)

48
main_package/tests/test_parameters_estimator.py
Unescape View File

@ -17,10 +17,14 @@ class TestParametersEstimatior(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls) -> None: def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') cls.array_indx = 8
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name',
cls.array_indx)
cls.s1 = sp.SamplePath(cls.importer) cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories() cls.s1.build_trajectories()
cls.s1.build_structure() cls.s1.build_structure()
print(cls.s1.structure.edges)
print(cls.s1.structure.nodes_values)
cls.g1 = ng.NetworkGraph(cls.s1.structure) cls.g1 = ng.NetworkGraph(cls.s1.structure)
cls.g1.init_graph() cls.g1.init_graph()
@ -28,29 +32,26 @@ class TestParametersEstimatior(unittest.TestCase):
for node in self.g1.nodes: for node in self.g1.nodes:
g = ng.NetworkGraph(self.s1.structure) g = ng.NetworkGraph(self.s1.structure)
g.fast_init(node) g.fast_init(node)
p1 = pe.ParametersEstimator(self.s1, g) p1 = pe.ParametersEstimator(self.s1.trajectories, g)
self.assertEqual(p1.sample_path, self.s1) self.assertEqual(p1._trajectories, self.s1.trajectories)
self.assertEqual(p1.net_graph, g) self.assertEqual(p1._net_graph, g)
self.assertIsNone(p1.single_set_of_cims) self.assertIsNone(p1._single_set_of_cims)
p1.fast_init(node) p1.fast_init(node)
self.assertIsInstance(p1.single_set_of_cims, sofc.SetOfCims) self.assertIsInstance(p1._single_set_of_cims, sofc.SetOfCims)
def test_compute_parameters_for_node(self): def test_compute_parameters_for_node(self):
for indx, node in enumerate(self.g1.nodes): for indx, node in enumerate(self.g1.nodes):
print(node) print(node)
g = ng.NetworkGraph(self.s1.structure) g = ng.NetworkGraph(self.s1.structure)
g.fast_init(node) g.fast_init(node)
p1 = pe.ParametersEstimator(self.s1, g) p1 = pe.ParametersEstimator(self.s1.trajectories, g)
p1.fast_init(node) p1.fast_init(node)
sofc1 = p1.compute_parameters_for_node(node) sofc1 = p1.compute_parameters_for_node(node)
sampled_cims = self.aux_import_sampled_cims('dyn.cims') sampled_cims = self.aux_import_sampled_cims('dyn.cims')
sc = list(sampled_cims.values()) sc = list(sampled_cims.values())
#print(sc[indx]) self.equality_of_cims_of_node(sc[indx], sofc1._actual_cims)
self.equality_of_cims_of_node(sc[indx], sofc1.actual_cims)
def equality_of_cims_of_node(self, sampled_cims, estimated_cims): def equality_of_cims_of_node(self, sampled_cims, estimated_cims):
#print(sampled_cims)
#print(estimated_cims)
self.assertEqual(len(sampled_cims), len(estimated_cims)) self.assertEqual(len(sampled_cims), len(estimated_cims))
for c1, c2 in zip(sampled_cims, estimated_cims): for c1, c2 in zip(sampled_cims, estimated_cims):
self.cim_equality_test(c1, c2.cim) self.cim_equality_test(c1, c2.cim)
@ -60,30 +61,9 @@ class TestParametersEstimatior(unittest.TestCase):
self.assertTrue(np.all(np.isclose(r1, r2, 1e-01, 1e-01) == True)) self.assertTrue(np.all(np.isclose(r1, r2, 1e-01, 1e-01) == True))
def aux_import_sampled_cims(self, cims_label): def aux_import_sampled_cims(self, cims_label):
i1 = ji.JsonImporter(self.read_files[0], '', '', '', '', '') i1 = ji.JsonImporter(self.read_files[0], '', '', '', '', '', self.array_indx)
raw_data = i1.read_json_file() raw_data = i1.read_json_file()
return i1.import_sampled_cims(raw_data, 0, cims_label) return i1.import_sampled_cims(raw_data, self.array_indx, cims_label)
"""
def test_init(self):
self.aux_test_init(self.s1, self.g1)
def test_init_sets_of_cims_container(self):
self.aux_test_init_sets_cims_container(self.s1, self.g1)
def aux_test_init(self, sample_p, graph):
pe1 = pe.ParametersEstimator(sample_p, graph)
self.assertEqual(sample_p, pe1.sample_path)
self.assertEqual(graph, pe1.net_graph)
self.assertIsNone(pe1.sets_of_cims_struct)
def aux_test_init_sets_cims_container(self, sample_p, graph):
pe1 = pe.ParametersEstimator(sample_p, graph)
pe1.init_sets_cims_container()
self.assertIsInstance(pe1.sets_of_cims_struct, scc.SetsOfCimsContainer)
def test_compute_parameters(self):
self.aux_test_compute_parameters(self.s1, self.g1)
"""
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

8
main_package/tests/test_sample_path.py
Unescape View File

@ -14,14 +14,14 @@ class TestSamplePath(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls) -> None: def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
def test_init(self): def test_init(self):
s1 = sp.SamplePath(self.importer) s1 = sp.SamplePath(self.importer)
self.assertIsNone(s1.trajectories) self.assertIsNone(s1.trajectories)
self.assertIsNone(s1.structure) self.assertIsNone(s1.structure)
self.assertFalse(s1.importer.concatenated_samples.empty) self.assertFalse(s1._importer.concatenated_samples.empty)
self.assertIsNone(s1.total_variables_count) self.assertIsNone(s1._total_variables_count)
def test_build_trajectories(self): def test_build_trajectories(self):
s1 = sp.SamplePath(self.importer) s1 = sp.SamplePath(self.importer)
@ -32,7 +32,7 @@ class TestSamplePath(unittest.TestCase):
s1 = sp.SamplePath(self.importer) s1 = sp.SamplePath(self.importer)
s1.build_structure() s1.build_structure()
self.assertIsInstance(s1.structure, st.Structure) self.assertIsInstance(s1.structure, st.Structure)
self.assertEqual(s1.total_variables_count, len(s1.importer.sorter)) self.assertEqual(s1._total_variables_count, len(s1._importer.sorter))
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

32
main_package/tests/test_setofcims.py
Unescape View File

@ -59,17 +59,17 @@ class TestSetOfCims(unittest.TestCase):
state_transition_matrix = np.random.randint(1, 10000, (3, 3)) state_transition_matrix = np.random.randint(1, 10000, (3, 3))
state_res_times_list.append(state_res_times) state_res_times_list.append(state_res_times)
transition_matrices_list.append(state_transition_matrix) transition_matrices_list.append(state_transition_matrix)
sofc1.build_cims(state_res_times_list, transition_matrices_list) sofc1.build_cims(np.array(state_res_times_list), np.array(transition_matrices_list))
for length_of_mask in range(3): for length_of_mask in range(3):
for mask in list(itertools.permutations([True, False],r=length_of_mask)): for mask in list(itertools.permutations([True, False],r=length_of_mask)):
m = np.array(mask) m = np.array(mask)
for parent_value in range(self.possible_cardinalities[0]): for parent_value in range(self.possible_cardinalities[0]):
cims = sofc1.filter_cims_with_mask(m, [parent_value]) cims = sofc1.filter_cims_with_mask(m, [parent_value])
if length_of_mask == 0 or length_of_mask == 1: if length_of_mask == 0 or length_of_mask == 1:
self.assertTrue(np.array_equal(sofc1.actual_cims, cims)) self.assertTrue(np.array_equal(sofc1._actual_cims, cims))
else: else:
indxs = self.another_filtering_method(p_combs, m, [parent_value]) indxs = self.another_filtering_method(p_combs, m, [parent_value])
self.assertTrue(np.array_equal(cims, sofc1.actual_cims[indxs])) self.assertTrue(np.array_equal(cims, sofc1._actual_cims[indxs]))
def aux_test_build_cims(self, node_id, p_values, node_states, p_combs): def aux_test_build_cims(self, node_id, p_values, node_states, p_combs):
state_res_times_list = [] state_res_times_list = []
@ -81,23 +81,23 @@ class TestSetOfCims(unittest.TestCase):
state_transition_matrix = np.random.randint(1, 10000, (node_states, node_states)) state_transition_matrix = np.random.randint(1, 10000, (node_states, node_states))
state_res_times_list.append(state_res_times) state_res_times_list.append(state_res_times)
transition_matrices_list.append(state_transition_matrix) transition_matrices_list.append(state_transition_matrix)
so1.build_cims(state_res_times_list, transition_matrices_list) so1.build_cims(np.array(state_res_times_list), np.array(transition_matrices_list))
self.assertEqual(len(state_res_times_list), so1.get_cims_number()) self.assertEqual(len(state_res_times_list), so1.get_cims_number())
self.assertIsInstance(so1.actual_cims, np.ndarray) self.assertIsInstance(so1._actual_cims, np.ndarray)
self.assertIsNone(so1.transition_matrices) self.assertIsNone(so1._transition_matrices)
self.assertIsNone(so1.state_residence_times) self.assertIsNone(so1._state_residence_times)
def aux_test_init(self, node_id, parents_states_number, node_states_number, p_combs): def aux_test_init(self, node_id, parents_states_number, node_states_number, p_combs):
sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number, p_combs) sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number, p_combs)
self.assertEqual(sofcims.node_id, node_id) self.assertEqual(sofcims._node_id, node_id)
self.assertTrue(np.array_equal(sofcims.p_combs, p_combs)) self.assertTrue(np.array_equal(sofcims._p_combs, p_combs))
self.assertTrue(np.array_equal(sofcims.parents_states_number, parents_states_number)) self.assertTrue(np.array_equal(sofcims._parents_states_number, parents_states_number))
self.assertEqual(sofcims.node_states_number, node_states_number) self.assertEqual(sofcims._node_states_number, node_states_number)
self.assertFalse(sofcims.actual_cims) self.assertFalse(sofcims._actual_cims)
self.assertEqual(sofcims.state_residence_times.shape[0], np.prod(np.array(parents_states_number))) self.assertEqual(sofcims._state_residence_times.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims.state_residence_times[0]),node_states_number) self.assertEqual(len(sofcims._state_residence_times[0]), node_states_number)
self.assertEqual(sofcims.transition_matrices.shape[0], np.prod(np.array(parents_states_number))) self.assertEqual(sofcims._transition_matrices.shape[0], np.prod(np.array(parents_states_number)))
self.assertEqual(len(sofcims.transition_matrices[0][0]), node_states_number) self.assertEqual(len(sofcims._transition_matrices[0][0]), node_states_number)
def aux_test_indexes_converter(self, node_id, parents_states_number, node_states_number): def aux_test_indexes_converter(self, node_id, parents_states_number, node_states_number):
sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number) sofcims = soci.SetOfCims(node_id, parents_states_number, node_states_number)

28
main_package/tests/test_structure_estimator.py
Unescape View File

@ -21,7 +21,7 @@ class TestStructureEstimator(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name') cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 3)
cls.s1 = sp.SamplePath(cls.importer) cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories() cls.s1.build_trajectories()
cls.s1.build_structure() cls.s1.build_structure()
@ -30,14 +30,14 @@ class TestStructureEstimator(unittest.TestCase):
exp_alfa = 0.1 exp_alfa = 0.1
chi_alfa = 0.1 chi_alfa = 0.1
se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa) se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa)
self.assertEqual(self.s1, se1.sample_path) self.assertEqual(self.s1, se1._sample_path)
self.assertTrue(np.array_equal(se1.nodes, np.array(self.s1.structure.nodes_labels))) self.assertTrue(np.array_equal(se1._nodes, np.array(self.s1.structure.nodes_labels)))
self.assertTrue(np.array_equal(se1.nodes_indxs, self.s1.structure.nodes_indexes)) self.assertTrue(np.array_equal(se1._nodes_indxs, self.s1.structure.nodes_indexes))
self.assertTrue(np.array_equal(se1.nodes_vals, self.s1.structure.nodes_values)) self.assertTrue(np.array_equal(se1._nodes_vals, self.s1.structure.nodes_values))
self.assertEqual(se1.exp_test_sign, exp_alfa) self.assertEqual(se1._exp_test_sign, exp_alfa)
self.assertEqual(se1.chi_test_alfa, chi_alfa) self.assertEqual(se1._chi_test_alfa, chi_alfa)
self.assertIsInstance(se1.complete_graph, nx.DiGraph) self.assertIsInstance(se1._complete_graph, nx.DiGraph)
self.assertIsInstance(se1.cache, ch.Cache) self.assertIsInstance(se1._cache, ch.Cache)
def test_build_complete_graph(self): def test_build_complete_graph(self):
exp_alfa = 0.1 exp_alfa = 0.1
@ -76,17 +76,18 @@ class TestStructureEstimator(unittest.TestCase):
lp_wrapper = lp(se1.ctpc_algorithm) lp_wrapper = lp(se1.ctpc_algorithm)
lp_wrapper() lp_wrapper()
lp.print_stats() lp.print_stats()
print(se1.complete_graph.edges) print(se1._complete_graph.edges)
print(self.s1.structure.edges) print(self.s1.structure.edges)
for ed in self.s1.structure.edges: for ed in self.s1.structure.edges:
self.assertIn(tuple(ed), se1.complete_graph.edges) self.assertIn(tuple(ed), se1._complete_graph.edges)
tuples_edges = [tuple(rec) for rec in self.s1.structure.edges] tuples_edges = [tuple(rec) for rec in self.s1.structure.edges]
spurious_edges = [] spurious_edges = []
for ed in se1.complete_graph.edges: for ed in se1._complete_graph.edges:
if not(ed in tuples_edges): if not(ed in tuples_edges):
spurious_edges.append(ed) spurious_edges.append(ed)
print("Spurious Edges:",spurious_edges) print("Spurious Edges:",spurious_edges)
se1.save_results() print("Adj Matrix:", nx.adj_matrix(se1._complete_graph).toarray().astype(bool))
#se1.save_results()
def test_memory(self): def test_memory(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1) se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
@ -95,5 +96,6 @@ class TestStructureEstimator(unittest.TestCase):
mem = current_process.memory_info().rss mem = current_process.memory_info().rss
print("Average Memory Usage in MB:", mem / 10**6) print("Average Memory Usage in MB:", mem / 10**6)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()