1
0
Fork 0

Refactor DOCStrings to Sphinx sinthax;add an example of csv importer; Refactor NetworkGraph class and test;

parallel_struct_est
philpMartin 4 years ago
parent 5321999e5e
commit 1b6e94853e
  1. 9
      main_package/basic_main.py
  2. 127
      main_package/classes/abstract_importer.py
  3. 3
      main_package/classes/abstract_sample_path.py
  4. 54
      main_package/classes/cache.py
  5. 25
      main_package/classes/conditional_intensity_matrix.py
  6. 166
      main_package/classes/json_importer.py
  7. 196
      main_package/classes/network_graph.py
  8. 116
      main_package/classes/parameters_estimator.py
  9. 46
      main_package/classes/sample_path.py
  10. 75
      main_package/classes/set_of_cims.py
  11. 16
      main_package/classes/simple_cvs_importer.py
  12. 56
      main_package/classes/structure.py
  13. 159
      main_package/classes/structure_estimator.py
  14. 37
      main_package/classes/trajectory.py
  15. 160
      main_package/tests/test_networkgraph.py
  16. 11
      main_package/tests/test_structure_estimator.py

@ -1,9 +1,10 @@
import glob
import os
import sys
sys.path.append("./classes/")
import glob
import os
import network_graph as ng
import sample_path as sp
import parameters_estimator as pe
@ -24,7 +25,7 @@ def main():
#From The Structure Object build the Graph
g = ng.NetworkGraph(s1.structure)
#Select a node you want to estimate the parameters
node = g.nodes[1]
node = g.nodes[2]
print("NOde", node)
#Init the _graph specifically for THIS node
g.fast_init(node)

@ -1,25 +1,24 @@
import sys
sys.path.append("./classes/")
from abc import ABC, abstractmethod
import pandas as pd
import typing
class AbstractImporter(ABC):
"""
Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
"""Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
:_file_path: the file path
:_concatenated_samples: the concatenation of all the processed trajectories
:param file_path: the file path
:type file_path: str
:_concatenated_samples: Dataframe containing the concatenation of all the processed trajectories
:_df_structure: Dataframe containing the structure of the network (edges)
:_df_variables: Dataframe containing the nodes cardinalities
:_df_concatenated_samples: the concatenation and processing of all the trajectories present
in the list df_samples list
:_sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples
:_sorter: A list containing the columns header (excluding the time column) of the `_concatenated_samples`
"""
def __init__(self, file_path: str):
"""
Parameters:
:_file_path: the path to the file containing the data
"""Constructor
"""
self._file_path = file_path
self._df_variables = None
@ -29,46 +28,48 @@ class AbstractImporter(ABC):
super().__init__()
@abstractmethod
def import_data(self):
"""
Imports and prepares all data present needed for susequent computation.
Parameters:
:void
Returns:
:void
post[self]: the class members self._df_variables and self._df_structure HAVE to be properly constructed
def import_data(self) -> None:
"""Imports all the trajectories, variables cardinalities, and net edges.
.. warning::
The class members ``_df_variables`` and ``_df_structure`` HAVE to be properly constructed
as Pandas Dataframes with the following structure:
Header of self._df_structure = [From_Node | To_Node]
Header of self.df_variables = [Variable_Label | Variable_Cardinality]
Header of _df_structure = [From_Node | To_Node]
Header of _df_variables = [Variable_Label | Variable_Cardinality]
.. note::
See :class:``JsonImporter`` for an example of implementation of this method.
"""
pass
@abstractmethod
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
"""
Initializes the self._sorter class member from a trajectory dataframe, exctracting the header of the frame
"""Initializes the ``_sorter`` class member from a trajectory dataframe, exctracting the header of the frame
and keeping ONLY the variables symbolic labels, cutting out the time label in the header.
Parameters:
:sample_frame: The dataframe from which extract the header
Returns:
:a list containing the processed header.
:param sample_frame: The dataframe from which extract the header
:type sample_frame: pandas.DataFrame
:return: A list containing the processed header.
:rtype: List
"""
pass
def compute_row_delta_sigle_samples_frame(self, sample_frame: pd.DataFrame,
columns_header: typing.List, shifted_cols_header: typing.List) \
-> pd.DataFrame:
"""
Computes the difference between each value present in th time column.
"""Computes the difference between each value present in th time column.
Copies and shift by one position up all the values present in the remaining columns.
Parameters:
:sample_frame: the traj to be processed
:time_header_label: the label for the times
:columns_header: the original header of sample_frame
:shifted_cols_header: a copy of columns_header with changed names of the contents
Returns:
:sample_frame: the processed dataframe
pre: the Dataframe sample_frame has to follow the column structure of this header:
:param sample_frame: the traj to be processed
:type sample_frame: pandas.Dataframe
:param columns_header: the original header of sample_frame
:type columns_header: List
:param shifted_cols_header: a copy of columns_header with changed names of the contents
:type shifted_cols_header: List
:return: The processed dataframe
:rtype: pandas.Dataframe
.. warning::
the Dataframe ``sample_frame`` has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values]
"""
sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1)
@ -78,21 +79,23 @@ class AbstractImporter(ABC):
sample_frame.drop(sample_frame.tail(1).index, inplace=True)
return sample_frame
def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List):
"""
Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list _df_samples_list.
Concatenates the result in the dataframe concatanated_samples
Parameters:
time_header_label: the label of the time column
df_samples_list: the datframe's list to be processed and concatenated
Returns:
void
pre: the Dataframe sample_frame has to follow the column structure of this header:
def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List) -> None:
"""Calls the method ``compute_row_delta_sigle_samples_frame`` on every dataframe present in the list
``df_samples_list``.
Concatenates the result in the dataframe ``concatanated_samples``
:param df_samples_list: the datframe's list to be processed and concatenated
:type df_samples_list: List
.. warning::
The Dataframe sample_frame has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values]
The class member self._sorter HAS to be properly INITIALIZED (See class members definition doc)
.. note::
After the call of this method the class member ``concatanated_samples`` will contain all processed
and merged trajectories
"""
if not self.sorter:
if not self._sorter:
raise RuntimeError("The class member self._sorter has to be INITIALIZED!")
shifted_cols_header = [s + "S" for s in self._sorter]
compute_row_delta = self.compute_row_delta_sigle_samples_frame
@ -105,26 +108,28 @@ class AbstractImporter(ABC):
self._concatenated_samples = self._concatenated_samples[complete_header]
def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List:
"""
Builds a List containing the columns of dataframe and converts them to a numpy array.
Parameters:
:data_frame: the dataframe from which the columns have to be extracted and converted
Returns:
:columns_list: the resulting list of numpy arrays
"""Builds a List containing the columns of data_frame and converts them to a numpy array.
:param data_frame: the dataframe from which the columns have to be extracted and converted
:type data_frame: pandas.Dataframe
:return: the resulting list of numpy arrays
:rtype: List
"""
columns_list = [data_frame[column].to_numpy() for column in data_frame]
return columns_list
def clear_concatenated_frame(self):
"""
Removes all values in the dataframe concatenated_samples
Parameters:
:void
Returns:
:void
def clear_concatenated_frame(self) -> None:
"""Removes all values in the dataframe concatenated_samples.
"""
self._concatenated_samples = self._concatenated_samples.iloc[0:0]
@abstractmethod
def dataset_id(self) -> object:
"""If the original dataset contains multiple dataset, this method returns a unique id to identify the current
dataset
"""
pass
@property
def concatenated_samples(self) -> pd.DataFrame:
return self._concatenated_samples

@ -1,7 +1,10 @@
from abc import ABC, abstractmethod
import abstract_importer as ai
import sys
sys.path.append('.')
class AbstractSamplePath(ABC):

@ -1,62 +1,54 @@
import sys
sys.path.append("./classes/")
import typing
import set_of_cims as sofc
class Cache:
"""
This class has the role of a cache for SetOfCIMS of a test node that have been already computed during the ctpc algorithm.
"""This class acts as a cache of ``SetOfCims`` objects for a node.
:_list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related
:_list_of_sets_of_parents: a list of ``Sets`` objects of the parents to which the cim in cache at SAME
index is related
:_actual_cache: a list of setOfCims objects
"""
def __init__(self):
"""Constructor Method
"""
self._list_of_sets_of_parents = []
self._actual_cache = []
def find(self, parents_comb: typing.Set) -> sofc.SetOfCims:
"""
Tries to find in cache given the symbolic parents combination parents_comb the SetOfCims related to that parents_comb.
Parameters:
parents_comb: the parents related to that SetOfCims
Returns:
A SetOfCims object if the parents_comb index is found in _list_of_sets_of_parents.
None otherwise.
Tries to find in cache given the symbolic parents combination ``parents_comb`` the ``SetOfCims``
related to that ``parents_comb``.
:param parents_comb: the parents related to that ``SetOfCims``
:type parents_comb: Set
:return: A ``SetOfCims`` object if the ``parents_comb`` index is found in ``_list_of_sets_of_parents``.
None otherwise.
:rtype: SetOfCims
"""
try:
#print("Cache State:", self.list_of_sets_of_indxs)
#print("Look For:", parents_comb)
result = self._actual_cache[self._list_of_sets_of_parents.index(parents_comb)]
print("CACHE HIT!!!!", parents_comb)
return result
except ValueError:
return None
def put(self, parents_comb: typing.Union[typing.Set, str], socim: sofc.SetOfCims):
"""
Place in cache the SetOfCims object, and the related sybolyc index parents_comb in _list_of_sets_of_parents
Parameters:
parents_comb: the symbolic set index
socim: the related SetOfCims object
def put(self, parents_comb: typing.Set, socim: sofc.SetOfCims) -> None:
"""Place in cache the ``SetOfCims`` object, and the related symbolic index ``parents_comb`` in
``_list_of_sets_of_parents``.
Returns:
void
:param parents_comb: the symbolic set index
:type parents_comb: Set
:param socim: the related SetOfCims object
:type socim: SetOfCims
"""
#print("Putting in _cache:", parents_comb)
self._list_of_sets_of_parents.append(parents_comb)
self._actual_cache.append(socim)
def clear(self):
"""
Clear the contents of both caches.
Parameters:
void
Returns:
void
def clear(self) -> None:
"""Clear the contents both of ``_actual_cache`` and ``_list_of_sets_of_parents``.
"""
del self._list_of_sets_of_parents[:]
del self._actual_cache[:]

@ -2,32 +2,25 @@ import numpy as np
class ConditionalIntensityMatrix:
"""
Abstracts the Conditional Intesity matrix of a node as aggregation of the state residence times vector
"""Abstracts the Conditional Intesity matrix of a node as aggregation of the state residence times vector
and state transition matrix and the actual CIM matrix.
:_state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix
:param state_residence_times: state residence times vector
:type state_residence_times: numpy.array
:param state_transition_matrix: the transitions count matrix
:type state_transition_matrix: numpy.ndArray
:_cim: the actual cim of the node
"""
def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array):
"""
Parameters:
:_state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix
"""Constructor Method
"""
self._state_residence_times = state_residence_times
self._state_transition_matrix = state_transition_matrix
self._cim = self.state_transition_matrix.astype(np.float64)
def compute_cim_coefficients(self):
"""
Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x]
Parameters:
void
Returns:
void
def compute_cim_coefficients(self) -> None:
"""Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x].
The class member ``_cim`` will contain the computed cim
"""
np.fill_diagonal(self._cim, self._cim.diagonal() * -1)
self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T

@ -1,4 +1,5 @@
import sys
sys.path.append("./classes/")
import json
import typing
import pandas as pd
@ -7,34 +8,33 @@ import abstract_importer as ai
class JsonImporter(ai.AbstractImporter):
"""
Implements the abstracts methods of AbstractImporter and adds all the necessary methods to process and prepare the data in json ext.
"""Implements the abstracts methods of AbstractImporter and adds all the necessary methods to process and prepare the data in json ext.
with the following structure:
[0]
|_ dyn.cims
|_ dyn.str
|_ samples
|_ variabels
:file_path: the path of the file that contains tha data to be imported
:_samples_label: the reference key for the samples in the trajectories
:_structure_label: the reference key for the structure of the network data
:_variables_label: the reference key for the cardinalites of the nodes data
:_time_key: the key used to identify the timestamps in each trajectory
:_variables_key: the key used to identify the names of the variables in the net
:_df_samples_list: a Dataframe list in which every df contains a trajectory
:param file_path: the path of the file that contains tha data to be imported
:type file_path: string
:param samples_label: the reference key for the samples in the trajectories
:type samples_label: string
:param structure_label: the reference key for the structure of the network data
:type structure_label: string
:param variables_label: the reference key for the cardinalites of the nodes data
:type variables_label: string
:param time_key: the key used to identify the timestamps in each trajectory
:type time_key: string
:param variables_key: the key used to identify the names of the variables in the net
:type variables_key: string
:param array_indx: the index of the outer JsonArray to exctract the data from
:type array_indx: int
:_df_samples_list: a Dataframe list in which every dataframe contains a trajectory
"""
def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str,
variables_key: str, array_indx: int):
"""
Parameters:
:file_path: the path of the file that contains tha data to be imported
:_samples_label: the reference key for the samples in the trajectories
:_structure_label: the reference key for the structure of the network data
:_variables_label: the reference key for the cardinalites of the nodes data
:_time_key: the key used to identify the timestamps in each trajectory
:_variables_key: the key used to identify the names of the variables in the net
:_array_indx: the index of the outer json array from which import all the data
"""Constructor method
"""
self._samples_label = samples_label
self._structure_label = structure_label
@ -45,13 +45,8 @@ class JsonImporter(ai.AbstractImporter):
self._array_indx = array_indx
super(JsonImporter, self).__init__(file_path)
def import_data(self):
"""
Imports and prepares all data present needed for subsequent processing.
Parameters:
:void
Returns:
_void
def import_data(self) -> None:
"""Implements the abstract method of :class:`AbstractImporter`
"""
raw_data = self.read_json_file()
self._df_samples_list = self.import_trajectories(raw_data)
@ -62,77 +57,71 @@ class JsonImporter(ai.AbstractImporter):
self._df_variables = self.import_variables(raw_data)
def import_trajectories(self, raw_data: typing.List) -> typing.List:
"""
Imports the trajectories in the list of dicts raw_data.
Parameters:
:raw_data: List of Dicts
Returns:
:List of dataframes containing all the trajectories
"""Imports the trajectories from the list of dicts ``raw_data``.
:param raw_data: List of Dicts
:type raw_data: List
:return: List of dataframes containing all the trajectories
:rtype: List
"""
return self.normalize_trajectories(raw_data, self._array_indx, self._samples_label)
def import_structure(self, raw_data: typing.List) -> pd.DataFrame:
"""
Imports in a dataframe the data in the list raw_data at the key _structure_label
"""Imports in a dataframe the data in the list raw_data at the key ``_structure_label``
Parameters:
:raw_data: the data
Returns:
:Daframe containg the starting node a ending node of every arc of the network
:param raw_data: List of Dicts
:type raw_data: List
:return: Dataframe containg the starting node a ending node of every arc of the network
:rtype: pandas.Dataframe
"""
return self.one_level_normalizing(raw_data, self._array_indx, self._structure_label)
def import_variables(self, raw_data: typing.List) -> pd.DataFrame:
"""
Imports the data in raw_data at the key _variables_label.
Sorts the row of the dataframe df_variables using the list sorter.
Parameters:
:raw_data: the data
:sorter: the header of the dataset containing only variables symbolic labels
Returns:
:Datframe containg the variables simbolic labels and their cardinalities
"""Imports the data in ``raw_data`` at the key ``_variables_label``.
:param raw_data: List of Dicts
:type raw_data: List
:return: Datframe containg the variables simbolic labels and their cardinalities
:rtype: pandas.Dataframe
"""
return self.one_level_normalizing(raw_data, self._array_indx, self._variables_label)
def read_json_file(self) -> typing.List:
"""
Reads the JSON file in the path self.filePath
Parameters:
:void
Returns:
:data: the contents of the json file
"""Reads the JSON file in the path self.filePath
:return: The contents of the json file
:rtype: List
"""
with open(self._file_path) as f:
data = json.load(f)
return data
def one_level_normalizing(self, raw_data: typing.List, indx: int, key: str) -> pd.DataFrame:
"""
Extracts the one-level nested data in the list raw_data at the index indx at the key key
Parameters:
:raw_data: List of Dicts
:indx: The index of the array from which the data have to be extracted
:key: the key for the Dicts from which exctract data
Returns:
:a normalized dataframe:
"""Extracts the one-level nested data in the list ``raw_data`` at the index ``indx`` at the key ``key``.
:param raw_data: List of Dicts
:type raw_data: List
:param indx: The index of the array from which the data have to be extracted
:type indx: int
:param key: the key for the Dicts from which exctract data
:type key: string
:return: A normalized dataframe
:rtype: pandas.Datframe
"""
return pd.DataFrame(raw_data[indx][key])
def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str) -> typing.List:
"""
Extracts the traj in raw_data at the index index at the key trajectories key.
Parameters:
:raw_data: the data
:indx: the index of the array from which extract data
:trajectories_key: the key of the trajectories objects
Returns:
:A list of daframes containg the trajectories
Extracts the trajectories in ``raw_data`` at the index ``index`` at the key ``trajectories key``.
:param raw_data: List of Dicts
:type raw_data: List
:param indx: The index of the array from which the data have to be extracted
:type indx: int
:param trajectories_key: the key of the trajectories objects
:type trajectories_key: string
:return: A list of daframes containg the trajectories
:rtype: List
"""
dataframe = pd.DataFrame
smps = raw_data[indx][trajectories_key]
@ -140,34 +129,33 @@ class JsonImporter(ai.AbstractImporter):
return df_samples_list
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
"""
Implements the abstract method build_sorter for this dataset
"""Implements the abstract method build_sorter of the :class:`AbstractImporter` for this dataset
"""
columns_header = list(sample_frame.columns.values)
columns_header.remove(self._time_key)
return columns_header
def clear_data_frame_list(self):
"""
Removes all values present in the dataframes in the list _df_samples_list
Parameters:
:void
Returns:
:void
def clear_data_frame_list(self) -> None:
"""Removes all values present in the dataframes in the list ``_df_samples_list``.
"""
for indx in range(len(self._df_samples_list)):
self._df_samples_list[indx] = self._df_samples_list[indx].iloc[0:0]
def dataset_id(self) -> object:
return self._array_indx
def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict:
"""
Imports the synthetic CIMS in the dataset in a dictionary, using variables labels
"""Imports the synthetic CIMS in the dataset in a dictionary, using variables labels
as keys for the set of CIMS of a particular node.
Parameters:
:raw_data: the data
:indx: the json array index
:cims_key: the key where the json object cims are placed
Returns:
:a dictionary containing the sampled CIMS for all the variables in the net
:param raw_data: List of Dicts
:type raw_data: List
:param indx: The index of the array from which the data have to be extracted
:type indx: int
:param cims_key: the key where the json object cims are placed
:type cims_key: string
:return: a dictionary containing the sampled CIMS for all the variables in the net
:rtype: Dictionary
"""
cims_for_all_vars = {}
for var in raw_data[indx][cims_key]:
@ -179,5 +167,3 @@ class JsonImporter(ai.AbstractImporter):

@ -1,62 +1,54 @@
import sys
sys.path.append('./classes/')
import typing
import structure as st
import networkx as nx
import numpy as np
import structure as st
class NetworkGraph:
"""
Abstracts the infos contained in the Structure class in the form of a directed _graph.
Has the task of creating all the necessary filtering structures for parameters estimation
:_graph_struct: the Structure object from which infos about the net will be extracted
:_graph: directed _graph
:nodes_labels: the symbolic names of the variables
:nodes_indexes: the indexes of the nodes
:nodes_values: the cardinalites of the nodes
:_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every
node in the net
:_fancy_indexing: the indexes of every parent of every node in the net
"""Abstracts the infos contained in the Structure class in the form of a directed graph.
Has the task of creating all the necessary filtering and indexing structures for parameters estimation
:param graph_struct: the ``Structure`` object from which infos about the net will be extracted
:type graph_struct: Structure
:_graph: directed graph
:_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos
about every parents of the node of which all the indexing and filtering structures will be constructed.
:_time_scalar_indexing_structure: the indexing structure for state res time estimation
:_transition_scalar_indexing_structure: the indexing structure for transition computation
:_time_filtering: the columns filtering structure used in the computation of the state res times
:_transition_filtering: the columns filtering structure used in the computation of the transition
from one state to another
:self._p_combs_structure: all the possible parents states combination for every node in the net
:_p_combs_structure: all the possible parents states combination for the node of interest
"""
def __init__(self, graph_struct: st.Structure):
"""
Parameters:
:graph_struct:the Structure object from which infos about the net will be extracted
"""Constructor Method
"""
self._graph_struct = graph_struct
self._graph = nx.DiGraph()
self._nodes_indexes = self._graph_struct.nodes_indexes
self._nodes_labels = self._graph_struct.nodes_labels
self._nodes_values = self._graph_struct.nodes_values
self._aggregated_info_about_nodes_parents = None
self._fancy_indexing = None
self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None
self._time_filtering = None
self._transition_filtering = None
self._p_combs_structure = None
def fast_init(self, node_id: str):
"""
Initializes all the necessary structures for parameters estimation of the node identified by the label node_id
Parameters:
node_id: the label of the node
Returns:
void
def fast_init(self, node_id: str) -> None:
"""Initializes all the necessary structures for parameters estimation of the node identified by the label
node_id
:param node_id: the label of the node
:type node_id: string
"""
self.add_nodes(self._nodes_labels)
self.add_nodes(self._graph_struct.nodes_labels)
self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
self._fancy_indexing = self._aggregated_info_about_nodes_parents[1]
p_indxs = self._fancy_indexing
p_indxs = self._aggregated_info_about_nodes_parents[1]
p_vals = self._aggregated_info_about_nodes_parents[2]
self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id,
p_vals)
@ -67,48 +59,39 @@ class NetworkGraph:
self._transition_filtering = self.build_transition_filtering_for_a_node(node_indx, p_indxs)
self._p_combs_structure = self.build_p_comb_structure_for_a_node(p_vals)
def add_nodes(self, list_of_nodes: typing.List):
"""
Adds the nodes to the _graph contained in the list of nodes list_of_nodes.
def add_nodes(self, list_of_nodes: typing.List) -> None:
"""Adds the nodes to the ``_graph`` contained in the list of nodes ``list_of_nodes``.
Sets all the properties that identify a nodes (index, positional index, cardinality)
Parameters:
list_of_nodes: the nodes to add to _graph
Returns:
void
:param list_of_nodes: the nodes to add to ``_graph``
:type list_of_nodes: List
"""
nodes_indxs = self._nodes_indexes
nodes_indxs = self._graph_struct.nodes_indexes
nodes_vals = self._graph_struct.nodes_values
pos = 0
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1
def add_edges(self, list_of_edges: typing.List):
"""
Add the edges to the _graph contained in the list list_of_edges.
def add_edges(self, list_of_edges: typing.List) -> None:
"""Add the edges to the ``_graph`` contained in the list ``list_of_edges``.
Parameters:
list_of_edges
Returns:
void
:param list_of_edges: the list containing of tuples containing the edges
:type list_of_edges: List
"""
self._graph.add_edges_from(list_of_edges)
def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple:
"""
Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
"""Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
(parents_labels, parents_indexes, parents_cardinalities).
N.B. The parent set is sorted using the list of sorted nodes nodes
Parameters:
node: the label of the node
Returns:
a tuple containing all the parent set infos
:param node: the label of the node
:type node: string
:return: a tuple containing all the parent set infos
:rtype: Tuple
"""
parents = self.get_parents_by_id(node)
nodes = self._nodes_labels
nodes = self._graph_struct.nodes_labels
d = {v: i for i, v in enumerate(nodes)}
sorted_parents = sorted(parents, key=lambda v: d[v])
get_node_indx = self.get_node_indx
@ -117,15 +100,14 @@ class NetworkGraph:
return (sorted_parents, p_indxes, p_values)
def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray:
"""
Builds an indexing structure for the computation of state residence times values.
Parameters:
node_id: the node label
parents_vals: the caridinalites of the node's parents
Returns:
a numpy array.
"""Builds an indexing structure for the computation of state residence times values.
:param node_id: the node label
:type node_id: string
:param parents_vals: the caridinalites of the node's parents
:type parents_vals: List
:return: The time indexing structure
:rtype: numpy.ndArray
"""
T_vector = np.array([self.get_states_number(node_id)])
T_vector = np.append(T_vector, parents_vals)
@ -134,15 +116,14 @@ class NetworkGraph:
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) \
-> np.ndarray:
"""
Builds an indexing structure for the computation of state transitions values.
Parameters:
node_id: the node label
parents_vals: the caridinalites of the node's parents
Returns:
a numpy array.
"""Builds an indexing structure for the computation of state transitions values.
:param node_id: the node label
:type node_id: string
:param parents_vals: the caridinalites of the node's parents
:type parents_vals: List
:return: The transition indexing structure
:rtype: numpy.ndArray
"""
node_states_number = self.get_states_number(node_id)
M_vector = np.array([node_states_number,
@ -153,37 +134,41 @@ class NetworkGraph:
def build_time_columns_filtering_for_a_node(self, node_indx: int, p_indxs: typing.List) -> np.ndarray:
"""
Builds the necessary structure to filter the desired columns indicated by node_indx and p_indxs in the dataset.
Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
in the dataset.
This structute will be used in the computation of the state res times.
Parameters:
node_indx: the index of the node
p_indxs: the indexes of the node's parents
Returns:
a numpy array
:param node_indx: the index of the node
:type node_indx: int
:param p_indxs: the indexes of the node's parents
:type p_indxs: List
:return: The filtering structure for times estimation
:rtype: numpy.ndArray
"""
return np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
def build_transition_filtering_for_a_node(self, node_indx, p_indxs) -> np.ndarray:
"""
Builds the necessary structure to filter the desired columns indicated by node_indx and p_indxs in the dataset.
This structute will be used in the computation of the state transitions values.
Parameters:
node_indx: the index of the node
p_indxs: the indexes of the node's parents
Returns:
a numpy array
def build_transition_filtering_for_a_node(self, node_indx: int, p_indxs: typing.List) -> np.ndarray:
"""Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
in the dataset.
This structure will be used in the computation of the state transitions values.
:param node_indx: the index of the node
:type node_indx: int
:param p_indxs: the indexes of the node's parents
:type p_indxs: List
:return: The filtering structure for transitions estimation
:rtype: numpy.ndArray
"""
nodes_number = self._graph_struct.total_variables_number
return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray:
"""
Builds the combinatory structure that contains the combinations of all the values contained in parents_values.
Builds the combinatorial structure that contains the combinations of all the values contained in
``parents_values``.
Parameters:
parents_values: the cardinalities of the nodes
Returns:
a numpy matrix containinga grid of the combinations
:param parents_values: the cardinalities of the nodes
:type parents_values: List
:return: A numpy matrix containing a grid of the combinations
:rtype: numpy.ndArray
"""
tmp = []
for val in parents_values:
@ -198,21 +183,28 @@ class NetworkGraph:
parents_comb = np.array([[]], dtype=np.int)
return parents_comb
def get_parents_by_id(self, node_id):
def get_parents_by_id(self, node_id) -> typing.List:
"""Returns a list of labels of the parents of the node ``node_id``
:param node_id: the node label
:type node_id: string
:return: a List of labels of the parents
:rtype: List
"""
return list(self._graph.predecessors(node_id))
def get_states_number(self, node_id):
def get_states_number(self, node_id) -> int:
return self._graph.nodes[node_id]['val']
def get_node_indx(self, node_id):
def get_node_indx(self, node_id) -> int:
return nx.get_node_attributes(self._graph, 'indx')[node_id]
def get_positional_node_indx(self, node_id):
def get_positional_node_indx(self, node_id) -> int:
return self._graph.nodes[node_id]['pos_indx']
@property
def nodes(self) -> typing.List:
return self._nodes_labels
return self._graph_struct.nodes_labels
@property
def edges(self) -> typing.List:
@ -220,11 +212,11 @@ class NetworkGraph:
@property
def nodes_indexes(self) -> np.ndarray:
return self._nodes_indexes
return self._graph_struct.nodes_indexes
@property
def nodes_values(self) -> np.ndarray:
return self._nodes_values
return self._graph_struct.nodes_values
@property
def time_scalar_indexing_strucure(self) -> np.ndarray:
@ -246,7 +238,8 @@ class NetworkGraph:
def p_combs(self) -> np.ndarray:
return self._p_combs_structure
"""##############These Methods are actually unused but could become useful in the near future################"""
"""
##############These Methods are actually unused but could become useful in the near future################
def init_graph(self):
self.add_nodes(self._nodes_labels)
@ -303,3 +296,4 @@ class NetworkGraph:
else:
fancy_indx = [i[1] for i in self._aggregated_info_about_nodes_parents]
return fancy_indx
"""

@ -1,60 +1,52 @@
import sys
sys.path.append("./classes/")
import numpy as np
import network_graph as ng
import trajectory as tr
import set_of_cims as sofc
import sets_of_cims_container as acims
class ParametersEstimator:
"""
Has the task of computing the cims of particular node given the trajectories in samplepath and the net structure
in the graph _net_graph
"""Has the task of computing the cims of particular node given the trajectories and the net structure
in the graph ``_net_graph``.
:trajectories: the trajectories
:_net_graph: the net structure
:param trajectories: the trajectories
:type trajectories: Trajectory
:param net_graph: the net structure
:type net_graph: NetworkGraph
:_single_set_of_cims: the set of cims object that will hold the cims of the node
"""
def __init__(self, trajectories: tr.Trajectory, net_graph: ng.NetworkGraph):
"""Constructor Method
"""
Parameters:
:trajectories: the trajectories
:_net_graph: the net structure
"""
#self.sample_path = sample_path
self._trajectories = trajectories
self._net_graph = net_graph
#self.sets_of_cims_struct = None
self._single_set_of_cims = None
def fast_init(self, node_id: str):
"""
Initializes all the necessary structures for the parameters estimation.
def fast_init(self, node_id: str) -> None:
"""Initializes all the necessary structures for the parameters estimation for the node ``node_id``.
Parameters:
node_id: the node label
Returns:
void
:param node_id: the node label
:type node_id: string
"""
p_vals = self._net_graph._aggregated_info_about_nodes_parents[2]
node_states_number = self._net_graph.get_states_number(node_id)
self._single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs)
def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims:
"""
Compute the CIMS of the node identified by the label node_id
"""Compute the CIMS of the node identified by the label ``node_id``.
Parameters:
node_id: the node label
Returns:
A setOfCims object filled with the computed CIMS
:param node_id: the node label
:type node_id: string
:return: A SetOfCims object filled with the computed CIMS
:rtype: SetOfCims
"""
node_indx = self._net_graph.get_node_indx(node_id)
state_res_times = self._single_set_of_cims._state_residence_times
transition_matrices = self._single_set_of_cims._transition_matrices
#trajectory = self.sample_path.trajectories.trajectory
self.compute_state_res_time_for_node(node_indx, self._trajectories.times,
self._trajectories.trajectory,
self._net_graph.time_filtering,
@ -69,59 +61,54 @@ class ParametersEstimator:
return self._single_set_of_cims
def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray,
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray, T: np.ndarray):
"""
Compute the state residence times for a node and fill the matrix T with the results
Parameters:
node_indx: the index of the node
times: the times deltas vector
trajectory: the trajectory
cols_filter: the columns filtering structure
scalar_indexes_struct: the indexing structure
T: the state residence times vectors
Returns:
void
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray,
T: np.ndarray) -> None:
"""Compute the state residence times for a node and fill the matrix ``T`` with the results
:param node_indx: the index of the node
:type node_indx: int
:param times: the times deltas vector
:type times: numpy.array
:param trajectory: the trajectory
:type trajectory: numpy.ndArray
:param cols_filter: the columns filtering structure
:type cols_filter: numpy.array
:param scalar_indexes_struct: the indexing structure
:type scalar_indexes_struct: numpy.array
:param T: the state residence times vectors
:type T: numpy.ndArray
"""
T[:] = np.bincount(np.sum(trajectory[:, cols_filter] * scalar_indexes_struct / scalar_indexes_struct[0], axis=1)
.astype(np.int), \
times,
minlength=scalar_indexes_struct[-1]).reshape(-1, T.shape[1])
def compute_state_transitions_for_a_node(self, node_indx, trajectory, cols_filter, scalar_indexing, M):
"""
Compute the state residence times for a node and fill the matrices M with the results
Parameters:
node_indx: the index of the node
times: the times deltas vector
trajectory: the trajectory
cols_filter: the columns filtering structure
scalar_indexes: the indexing structure
M: the state transition matrices
Returns:
void
def compute_state_transitions_for_a_node(self, node_indx: int, trajectory: np.ndarray, cols_filter: np.ndarray,
scalar_indexing: np.ndarray, M: np.ndarray):
"""Compute the state residence times for a node and fill the matrices ``M`` with the results.
:param node_indx: the index of the node
:type node_indx: int
:param trajectory: the trajectory
:type trajectory: numpy.ndArray
:param cols_filter: the columns filtering structure
:type cols_filter: numpy.array
:param scalar_indexing: the indexing structure
:type scalar_indexing: numpy.array
:param M: the state transitions matrices
:type M: numpy.ndArray
"""
diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])],
dtype=np.int64)
trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0]
#print("Trajectory", trajectory)
#print("Step 1", trajectory[:, int(trajectory.shape[1] / 2) + node_indx])
#print("Step 2", trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0)
#print("TrTemp", trj_tmp)
#print("Cols Filter", cols_filter)
#print("Filtered Tr Temp", trj_tmp[:, cols_filter])
#print("Actual Indexing", scalar_indexing / scalar_indexing[0])
#print("PreBins",trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0] )
#print("Bins", np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1))
#print("After BinCount", np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int)))
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int),
minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
M_raveled = M.ravel()
M_raveled[diag_indices] = 0
M_raveled[diag_indices] = np.sum(M, axis=2).ravel()
"""##############These Methods are actually unused but could become useful in the near future################"""
"""
##############These Methods are actually unused but could become useful in the near future################
def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self._net_graph.nodes,
@ -143,6 +130,7 @@ class ParametersEstimator:
self._net_graph.transition_scalar_indexing_structure[indx],
aggr[1]._transition_matrices)
aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices)
"""

@ -3,23 +3,23 @@ import abstract_importer as imp
import structure as st
import trajectory as tr
import sys
sys.path.append('./classes/')
class SamplePath:
"""
Aggregates all the informations about the trajectories, the real structure of the sampled net and variables
cardinalites.
Has the task of creating the objects that will contain the mentioned data.
:_importer: the Importer objects that will import ad process data
:trajectories: the Trajectory object that will contain all the concatenated trajectories
:structure: the Structure Object that will contain all the structurral infos about the net
class SamplePath:
"""Aggregates all the informations about the trajectories, the real structure of the sampled net and variables
cardinalites. Has the task of creating the objects ``Trajectory`` and ``Structure`` that will
contain the mentioned data.
:param importer: the Importer objects that will import ad process data
:type importer: AbstractImporter
:_trajectories: the ``Trajectory`` object that will contain all the concatenated trajectories
:_structure: the ``Structure`` Object that will contain all the structurral infos about the net
:_total_variables_count: the number of variables in the net
"""
def __init__(self, importer: imp.AbstractImporter):
"""
Parameters:
:importer: the Importer objects that will import ad process data
"""Constructor Method
"""
self._importer = importer
self._trajectories = None
@ -27,30 +27,18 @@ class SamplePath:
self._total_variables_count = None
self._importer.import_data()
def build_trajectories(self):
"""
Builds the Trajectory object that will contain all the trajectories.
Clears all the unused dataframes in Importer Object
Parameters:
:void
Returns:
:void
def build_trajectories(self) -> None:
"""Builds the Trajectory object that will contain all the trajectories.
Clears all the unused dataframes in ``_importer`` Object
"""
#self._importer.import_data()
self._trajectories = \
tr.Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples),
len(self._importer.sorter) + 1)
#self.trajectories.append(trajectory)
self._importer.clear_concatenated_frame()
def build_structure(self):
def build_structure(self) -> None:
"""
Builds the Structure object that aggregates all the infos about the net.
Parameters:
:void
Returns:
:void
Builds the ``Structure`` object that aggregates all the infos about the net.
"""
if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list():
raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!")

@ -1,31 +1,29 @@
import typing
import numpy as np
import sys
sys.path.append("./classes/")
import conditional_intensity_matrix as cim
class SetOfCims:
"""
Aggregates all the CIMS of the node identified by the label _node_id.
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
:state_residence_time: matrix containing all the state residence time vectors for the node
"""Aggregates all the CIMS of the node identified by the label _node_id.
:param node_id: the node label
:type node_ind: string
:param parents_states_number: the cardinalities of the parents
:type parents_states_number: List
:param node_states_number: the caridinality of the node
:type node_states_number: int
:param p_combs: the p_comb structure bound to this node
:type p_combs: numpy.ndArray
:_state_residence_time: matrix containing all the state residence time vectors for the node
:_transition_matrices: matrix containing all the transition matrices for the node
:_actual_cims: the cims of the node
"""
def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray):
"""
Parameters:
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
"""Constructor Method
"""
self._node_id = node_id
self._parents_states_number = parents_states_number
@ -36,14 +34,8 @@ class SetOfCims:
self._p_combs = p_combs
self.build_times_and_transitions_structures()
def build_times_and_transitions_structures(self):
"""
Initializes at the correct dimensions the state residence times matrix and the state transition matrices
Parameters:
:void
Returns:
:void
def build_times_and_transitions_structures(self) -> None:
"""Initializes at the correct dimensions the state residence times matrix and the state transition matrices.
"""
if not self._parents_states_number:
self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float)
@ -54,16 +46,14 @@ class SetOfCims:
self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number,
self._node_states_number], dtype=np.int)
def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray):
"""
Build the ConditionalIntensityMatrix object given the state residence times and transitions matrices.
Compute the cim coefficients.
Parameters:
:state_res_times: the state residence times matrix
:_transition_matrices: the transition matrices
Returns:
:void
def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray) -> None:
"""Build the ``ConditionalIntensityMatrix`` objects given the state residence times and transitions matrices.
Compute the cim coefficients.The class member ``_actual_cims`` will contain the computed cims.
:param state_res_times: the state residence times matrix
:type state_res_times: numpy.ndArray
:param transition_matrices: the transition matrices
:type transition_matrices: numpy.ndArray
"""
for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices):
cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix)
@ -74,14 +64,15 @@ class SetOfCims:
self._state_residence_times = None
def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray:
"""
Filter the cims contained in the array _actual_cims given the boolean mask mask_arr and the index comb.
Parameters:
:mask_arr: the boolean mask
:comb: the indexes of the selected cims
Returns:
:Array of ConditionalIntensityMatrix
"""Filter the cims contained in the array ``_actual_cims`` given the boolean mask ``mask_arr`` and the index
``comb``.
:param mask_arr: the boolean mask that indicates which parent to consider
:type mask_arr: numpy.array
:param comb: the state/s of the filtered parents
:type comb: numpy.array
:return: Array of ``ConditionalIntensityMatrix`` objects
:rtype: numpy.array
"""
if mask_arr.size <= 1:
return self._actual_cims

@ -2,6 +2,8 @@ import pandas as pd
import glob
import os
import typing
import abstract_importer as ai
import sample_path as sp
@ -14,6 +16,7 @@ class CSVImporter(ai.AbstractImporter):
def import_data(self):
self.read_csv_file()
self._sorter = self.build_sorter(self._df_samples_list[0])
self.import_variables()
self.import_structure()
self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
@ -24,16 +27,15 @@ class CSVImporter(ai.AbstractImporter):
self._df_samples_list = [df]
def import_variables(self):
variables_list = list(self._df_samples_list[0].columns)[1:]
#wrong_vars_labels = ['Y','Z','X']
self._sorter = variables_list
values_list = [3 for var in variables_list]
# initialize list of lists
data = {'Name':variables_list, 'Value':values_list}
values_list = [3 for var in self._sorter]
# initialize dict of lists
data = {'Name':self._sorter, 'Value':values_list}
# Create the pandas DataFrame
self._df_variables = pd.DataFrame(data)
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
return list(sample_frame.columns)[1:]
def import_structure(self):
data = {'From':['X','Y','Z'], 'To':['Z','Z','Y']}
self._df_structure = pd.DataFrame(data)

@ -4,29 +4,26 @@ import numpy as np
class Structure:
"""Contains all the infos about the network structure(nodes labels, nodes caridinalites, edges, indexes)
:param nodes_labels_list: the symbolic names of the variables
:type nodes_labels_list: List
:param nodes_indexes_arr: the indexes of the nodes
:type nodes_indexes_arr: numpy.ndArray
:param nodes_vals_arr: the cardinalites of the nodes
:type nodes_vals_arr: numpy.ndArray
:param edges_list: the edges of the network
:type edges_list: List
:param total_variables_number: the total number of variables in the net
:type total_variables_number: int
"""
Contains all the infos about the network structure(nodes names, nodes caridinalites, edges...)
:nodes_labels_list: the symbolic names of the variables
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
def __init__(self, nodes_label_list: ty.List, node_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
def __init__(self, nodes_labels_list: ty.List, nodes_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
edges_list: ty.List, total_variables_number: int):
"""Constructor Method
"""
Parameters:
:nodes_labels_list: the symbolic names of the variables
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
self._nodes_labels_list = nodes_label_list
self._nodes_indexes_arr = node_indexes_arr
self._nodes_labels_list = nodes_labels_list
self._nodes_indexes_arr = nodes_indexes_arr
self._nodes_vals_arr = nodes_vals_arr
self._edges_list = edges_list
self._total_variables_number = total_variables_number
@ -52,9 +49,23 @@ class Structure:
return self._total_variables_number
def get_node_id(self, node_indx: int) -> str:
"""Given the ``node_index`` returns the node label.
:param node_indx: the node index
:type node_indx: int
:return: the node label
:rtype: string
"""
return self._nodes_labels_list[node_indx]
def get_node_indx(self, node_id: str) -> int:
"""Given the ``node_index`` returns the node label.
:param node_id: the node label
:type node_id: string
:return: the node index
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node_id)
return self._nodes_indexes_arr[pos_indx]
@ -62,6 +73,13 @@ class Structure:
return self._nodes_labels_list.index(node_id)
def get_states_number(self, node: str) -> int:
"""Given the node label ``node`` returns the cardinality of the node.
:param node: the node label
:type node: string
:return: the node cardinality
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node)
return self._nodes_vals_arr[pos_indx]

@ -1,8 +1,11 @@
import sys
sys.path.append("./classes/")
from tqdm import tqdm
import itertools
import json
import typing
import networkx as nx
import numpy as np
from networkx.readwrite import json_graph
@ -18,27 +21,23 @@ import structure as st
class StructureEstimator:
"""
Has the task of estimating the network structure given the trajectories in samplepath.
:_sample_path: the _sample_path object containing the trajectories and the real structure
:_exp_test_sign: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test
:_nodes: the _nodes labels
"""Has the task of estimating the network structure given the trajectories in ``samplepath``.
:param sample_path: the _sample_path object containing the trajectories and the real structure
:type sample_path: SamplePath
:param exp_test_alfa: the significance level for the exponential Hp test
:type exp_test_alfa: float
:param chi_test_alfa: the significance level for the chi Hp test
:type chi_test_alfa: float
:_nodes: the nodes labels
:_nodes_vals: the nodes cardinalities
:_nodes_indxs: the nodes indexes
:_complete_graph: the complete directed graph built using the nodes labels in nodes
:_cache: the _cache object
:_complete_graph: the complete directed graph built using the nodes labels in ``_nodes``
:_cache: the Cache object
"""
def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float):
"""
Parameters:
:_sample_path: the _sample_path object containing the trajectories and the real structure
:_exp_test_sign: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test
"""Constructor Method
"""
self._sample_path = sample_path
self._nodes = np.array(self._sample_path.structure.nodes_labels)
@ -50,13 +49,12 @@ class StructureEstimator:
self._cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List) -> nx.DiGraph:
"""
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
"""Builds a complete directed graph (no self loops) given the nodes labels in the list ``node_ids``:
Parameters:
node_ids: the list of nodes labels
Returns:
a complete Digraph Object
:param node_ids: the list of nodes labels
:type node_ids: List
:return: a complete Digraph Object
:rtype: networkx.DiGraph
"""
complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids)
@ -65,20 +63,22 @@ class StructureEstimator:
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int) -> bool:
"""
Performs a complete independence test on the directed graphs G1 = test_child U parent_set
G2 = G1 U test_parent (added as an additional parent of the test_child).
"""Performs a complete independence test on the directed graphs G1 = {test_child U parent_set}
G2 = {G1 U test_parent} (added as an additional parent of the test_child).
Generates all the necessary structures and datas to perform the tests.
Parameters:
test_parent: the node label of the test parent
test_child: the node label of the child
parent_set: the common parent set
child_states_numb: the cardinality of the test_child
tot_vars_count: the total number of variables in the net
Returns:
True iff test_child and test_parent are independent given the sep_set parent_set
False otherwise
:param test_parent: the node label of the test parent
:type test_parent: string
:param test_child: the node label of the child
:type test_child: string
:param parent_set: the common parent set
:type parent_set: List
:param child_states_numb: the cardinality of the ``test_child``
:type child_states_numb: int
:param tot_vars_count: the total number of variables in the net
:type tot_vars_count: int
:return: True iff test_child and test_parent are independent given the sep_set parent_set. False otherwise
:rtype: bool
"""
p_set = parent_set[:]
complete_info = parent_set[:]
@ -130,19 +130,18 @@ class StructureEstimator:
def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix,
cim2: condim.ConditionalIntensityMatrix) -> bool:
"""
Compute the actual independence test using two cims.
"""Compute the actual independence test using two cims.
It is performed first the exponential test and if the null hypothesis is not rejected,
it is permormed also the chi_test.
Parameters:
child_states_numb: the cardinality of the test child
cim1: a cim belonging to the graph without test parent
cim2: a cim belonging to the graph with test parent
it is performed also the chi_test.
Returns:
True iff both tests do NOT reject the null hypothesis of indipendence
False otherwise
:param child_states_numb: the cardinality of the test child
:type child_states_numb: int
:param cim1: a cim belonging to the graph without test parent
:type cim1: ConditionalIntensityMatrix
:param cim2: a cim belonging to the graph with test parent
:type cim2: ConditionalIntensityMatrix
:return:True iff both tests do NOT reject the null hypothesis of indipendence. False otherwise.
:rtype: bool
"""
M1 = cim1.state_transition_matrix
M2 = cim2.state_transition_matrix
@ -155,7 +154,6 @@ class StructureEstimator:
for val in range(0, child_states_numb):
if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \
F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]):
#print("CONDITIONALLY DEPENDENT EXP")
return False
M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1)
M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape(
@ -170,17 +168,15 @@ class StructureEstimator:
return False
return True
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int):
"""
Performs an iteration of the CTPC algorithm using the node var_id as test_child.
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int) -> None:
"""Performs an iteration of the CTPC algorithm using the node ``var_id`` as ``test_child``.
Parameters:
var_id: the node label of the test child
tot_vars_count: the number of _nodes in the net
Returns:
void
:param var_id: the node label of the test child
:type var_id: string
:param tot_vars_count: the number of _nodes in the net
:type tot_vars_count: int
"""
print("##################TESTING VAR################", var_id)
#print("##################TESTING VAR################", var_id)
u = list(self._complete_graph.predecessors(var_id))
child_states_numb = self._sample_path.structure.get_states_number(var_id)
b = 0
@ -190,12 +186,8 @@ class StructureEstimator:
removed = False
S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])
test_parent = u[parent_indx]
#print("Test Parent", test_parent)
for parents_set in S:
print("Parent Set", parents_set)
print("Test Parent", test_parent)
if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count):
#print("Removing EDGE:", test_parent, var_id)
self._complete_graph.remove_edge(test_parent, var_id)
u.remove(test_parent)
removed = True
@ -207,46 +199,37 @@ class StructureEstimator:
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str) -> \
typing.Iterator:
"""
Creates a list containing all possible subsets of the list u of size size,
that do not contains a the node identified by parent_label.
Parameters:
u: the list of _nodes
size: the size of the subsets
parent_label: the _nodes to exclude in the subsets generation
Returns:
a Map Object containing a list of lists
"""Creates a list containing all possible subsets of the list ``u`` of size ``size``,
that do not contains a the node identified by ``parent_label``.
:param u: the list of nodes
:type u: List
:param size: the size of the subsets
:type size: int
:param parent_label: the node to exclude in the subsets generation
:type parent_label: string
:return: an Iterator Object containing a list of lists
:rtype: Iterator
"""
list_without_test_parent = u[:]
list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size))
def ctpc_algorithm(self):
"""
Compute the CTPC algorithm.
Parameters:
void
Returns:
void
def ctpc_algorithm(self) -> None:
"""Compute the CTPC algorithm over the entire net.
"""
ctpc_algo = self.one_iteration_of_CTPC_algorithm
total_vars_numb = self._sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self._nodes]
[ctpc_algo(n, total_vars_numb) for n in tqdm(self._nodes)]
def save_results(self):
"""
Save the estimated Structure to a .json file
def save_results(self) -> None:
"""Save the estimated Structure to a .json file in the path where the data are loaded from.
The file is named as the input dataset but the results_ word is appendend to the results file.
Parameters:
void
Returns:
void
"""
res = json_graph.node_link_data(self._complete_graph)
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1]
name = 'results_' + name #TODO va aggiunto anche l'indice di array
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1] + str(self._sample_path._importer.dataset_id())
name = 'results_' + name
with open(name, 'w') as f:
json.dump(res, f)

@ -1,25 +1,22 @@
import numpy as np
import typing
class Trajectory:
"""
Abstracts the infos about a complete set of trajectories, represented as a numpy array of doubles and a numpy matrix
of ints.
:list_of_columns: the list containing the times array and values matrix
:original_cols_numb: total number of cols in the data
:actual_trajectory: the trajectory containing also the duplicated and shifted values
:times: the array containing the time deltas
"""
""" Abstracts the infos about a complete set of trajectories, represented as a numpy array of doubles (the time deltas)
and a numpy matrix of ints (the changes of states).
def __init__(self, list_of_columns, original_cols_number):
:param list_of_columns: the list containing the times array and values matrix
:type list_of_columns: List
:param original_cols_number: total number of cols in the data
:type original_cols_number: int
:_actual_trajectory: the trajectory containing also the duplicated/shifted values
:_times: the array containing the time deltas
"""
Parameters:
:list_of_columns: the list containing the times array and values matrix
:original_cols_numb: total number of cols in the data
def __init__(self, list_of_columns: typing.List, original_cols_number: int):
"""Constructor Method
"""
if type(list_of_columns[0][0]) != np.float64:
raise TypeError('The first array in the list has to be Times')
@ -29,22 +26,10 @@ class Trajectory:
@property
def trajectory(self) -> np.ndarray:
"""
Parameters:
void
Returns:
a numpy matrix containing ONLY the original columns values, not the shifted ones
"""
return self._actual_trajectory[:, :self._original_cols_number]
@property
def complete_trajectory(self) -> np.ndarray:
"""
Parameters:
void
Returns:
a numpy matrix containing all the values
"""
return self._actual_trajectory
@property

@ -25,10 +25,10 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure)
self.assertEqual(self.s1.structure, g1._graph_struct)
self.assertIsInstance(g1._graph, nx.DiGraph)
self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes))
self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels)
self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values))
self.assertIsNone(g1._fancy_indexing)
#self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes))
#self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels)
#self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values))
#self.assertIsNone(g1._fancy_indexing)
self.assertIsNone(g1.time_scalar_indexing_strucure)
self.assertIsNone(g1.transition_scalar_indexing_structure)
self.assertIsNone(g1.transition_filtering)
@ -46,28 +46,40 @@ class TestNetworkGraph(unittest.TestCase):
for e in self.s1.structure.edges:
self.assertIn(tuple(e), g1.edges)
def aux_aggregated_par_list_data(self, graph, node_id, sorted_par_list_aggregated_info):
for indx, element in enumerate(sorted_par_list_aggregated_info):
if indx == 0:
self.assertEqual(graph.get_parents_by_id(node_id), element)
for j in range(0, len(sorted_par_list_aggregated_info[0]) - 1):
self.assertLess(self.s1.structure.get_node_indx(sorted_par_list_aggregated_info[0][j]),
self.s1.structure.get_node_indx(sorted_par_list_aggregated_info[0][j + 1]))
elif indx == 1:
for node, node_indx in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[1]):
self.assertEqual(graph.get_node_indx(node), node_indx)
else:
for node, node_val in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[2]):
self.assertEqual(graph._graph_struct.get_states_number(node), node_val)
def test_get_ord_set_of_par_of_all_nodes(self):
def test_fast_init(self):
g1 = ng.NetworkGraph(self.s1.structure)
for node in self.s1.structure.nodes_labels:
g1.fast_init(node)
self.assertIsNotNone(g1._graph.nodes)
self.assertIsNotNone(g1._graph.edges)
self.assertIsInstance(g1._time_scalar_indexing_structure, np.ndarray)
self.assertIsInstance(g1._transition_scalar_indexing_structure, np.ndarray)
self.assertIsInstance(g1._time_filtering, np.ndarray)
self.assertIsInstance(g1._transition_filtering, np.ndarray)
self.assertIsInstance(g1._p_combs_structure, np.ndarray)
self.assertIsInstance(g1._aggregated_info_about_nodes_parents, tuple)
def test_get_ordered_by_indx_set_of_parents(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
sorted_list_of_par_lists = g1.get_ord_set_of_par_of_all_nodes()
for node, par_list in zip(g1.nodes, sorted_list_of_par_lists):
self.aux_aggregated_par_list_data(g1, node, par_list)
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
for indx in range(len(aggr_info[0]) - 1 ):
self.assertLess(g1.get_node_indx(aggr_info[0][indx]), g1.get_node_indx(aggr_info[0][indx + 1]))
for par, par_indx in zip(aggr_info[0], aggr_info[1]):
self.assertEqual(g1.get_node_indx(par), par_indx)
for par, par_val in zip(aggr_info[0], aggr_info[2]):
self.assertEqual(g1._graph_struct.get_states_number(par), par_val)
def test_build_time_scalar_indexing_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node, aggr_info[1],
aggr_info[0], aggr_info[2])
def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals):
time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals)
@ -81,6 +93,15 @@ class TestNetworkGraph(unittest.TestCase):
t_vec = t_vec.cumprod()
self.assertTrue(np.array_equal(time_scalar_indexing, t_vec))
def test_build_transition_scalar_indexing_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node, aggr_info[1],
aggr_info[0], aggr_info[2])
def aux_build_transition_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels,
parents_values):
transition_scalar_indexing = graph.build_transition_scalar_indexing_structure_for_a_node(node_id,
@ -96,93 +117,54 @@ class TestNetworkGraph(unittest.TestCase):
m_vec = m_vec.cumprod()
self.assertTrue(np.array_equal(transition_scalar_indexing, m_vec))
def test_build_transition_scalar_indexing_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
for node_id, p_i ,p_l, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node_id, p_i ,p_l, p_v)
def test_build_time_scalar_indexing_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
#print(fancy_indx)
for node_id, p_indxs, p_labels, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node_id, p_indxs, p_labels, p_v)
def test_build_time_columns_filtering_structure(self):
def test_build_time_columns_filtering_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_time_columns_filtering_structure()
t_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing):
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_time_columns_filtering_structure_for_a_node(g1, node, aggr_info[1])
def aux_build_time_columns_filtering_structure_for_a_node(self, graph, node_id, p_indxs):
graph.build_time_columns_filtering_for_a_node(graph.get_node_indx(node_id), p_indxs)
single_filter = []
single_filter.append(g1.get_node_indx(node_id))
single_filter.append(graph.get_node_indx(node_id))
single_filter.extend(p_indxs)
t_filter.append(np.array(single_filter))
#print(t_filter)
for a1, a2 in zip(g1.time_filtering, t_filter):
self.assertTrue(np.array_equal(a1, a2))
self.assertTrue(np.array_equal(graph.build_time_columns_filtering_for_a_node(graph.get_node_indx(node_id),
p_indxs),np.array(single_filter)))
def test_build_transition_columns_filtering_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_transition_columns_filtering_structure()
m_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing):
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_time_columns_filtering_structure_for_a_node(g1, node, aggr_info[1])
def aux_build_transition_columns_filtering_structure(self, graph, node_id, p_indxs):
single_filter = []
single_filter.append(g1.get_node_indx(node_id) + g1._graph_struct.total_variables_number)
single_filter.append(g1.get_node_indx(node_id))
single_filter.append(graph.get_node_indx(node_id) + graph._graph_struct.total_variables_number)
single_filter.append(graph.get_node_indx(node_id))
single_filter.extend(p_indxs)
m_filter.append(np.array(single_filter))
for a1, a2 in zip(g1.transition_filtering, m_filter):
self.assertTrue(np.array_equal(a1, a2))
self.assertTrue(np.array_equal(graph.build_transition_filtering_for_a_node(graph.get_node_indx(node_id),
p_indxs), np.array(single_filter)))
def test_build_p_combs_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
p_combs = g1.build_p_combs_structure()
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_p_combs_structure(g1, aggr_info[2])
for matrix, p_v in zip(p_combs, p_vals):
def aux_build_p_combs_structure(self, graph, p_vals):
p_combs = graph.build_p_comb_structure_for_a_node(p_vals)
p_possible_vals = []
for val in p_v:
for val in p_vals:
vals = [v for v in range(val)]
p_possible_vals.extend(vals)
comb_struct = set(itertools.product(p_possible_vals,repeat=len(p_v)))
#print(comb_struct)
comb_struct = set(itertools.product(p_possible_vals,repeat=len(p_vals)))
for comb in comb_struct:
self.assertIn(np.array(comb), matrix)
def test_fast_init(self):
g1 = ng.NetworkGraph(self.s1.structure)
g2 = ng.NetworkGraph(self.s1.structure)
g1.init_graph()
for indx, node in enumerate(g1.nodes):
g2.fast_init(node)
self.assertListEqual(g2._fancy_indexing, g1._fancy_indexing[indx])
self.assertTrue(np.array_equal(g2.time_scalar_indexing_strucure, g1.time_scalar_indexing_strucure[indx]))
self.assertTrue(np.array_equal(g2.transition_scalar_indexing_structure, g1.transition_scalar_indexing_structure[indx]))
self.assertTrue(np.array_equal(g2.time_filtering, g1.time_filtering[indx]))
self.assertTrue(np.array_equal(g2.transition_filtering, g1.transition_filtering[indx]))
self.assertTrue(np.array_equal(g2.p_combs, g1.p_combs[indx]))
self.assertIn(np.array(comb), p_combs)
def test_get_parents_by_id(self):
g1 = ng.NetworkGraph(self.s1.structure)

@ -9,6 +9,7 @@ import networkx as nx
import numpy as np
import psutil
from line_profiler import LineProfiler
import timeit
import cache as ch
import sample_path as sp
@ -21,7 +22,7 @@ class TestStructureEstimator(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 3)
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()
cls.s1.build_structure()
@ -70,12 +71,14 @@ class TestStructureEstimator(unittest.TestCase):
def test_time(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
lp = LineProfiler()
lp.add_function(se1.complete_test)
lp.add_function(se1.one_iteration_of_CTPC_algorithm)
lp.add_function(se1.independence_test)
#lp.add_function(se1.complete_test)
#lp.add_function(se1.one_iteration_of_CTPC_algorithm)
#lp.add_function(se1.independence_test)
lp_wrapper = lp(se1.ctpc_algorithm)
lp_wrapper()
lp.print_stats()
#print("Last time", lp.dump_stats())
#print("Exec Time", timeit.timeit(se1.ctpc_algorithm, number=1))
print(se1._complete_graph.edges)
print(self.s1.structure.edges)
for ed in self.s1.structure.edges: