unimib-thesis-mad
/
PyCTBN
Archived
1
0
Fork 0
Code Releases Activity

Refactor DOCStrings to Sphinx sinthax;add an example of csv importer; Refactor NetworkGraph class and test;

Browse Source
parallel_struct_est
philpMartin 4 years ago
parent 5321999e5e
commit 1b6e94853e
16 changed files with 598 additions and 684 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 9
      main_package/basic_main.py
  2. 129
      main_package/classes/abstract_importer.py
  3. 3
      main_package/classes/abstract_sample_path.py
  4. 54
      main_package/classes/cache.py
  5. 25
      main_package/classes/conditional_intensity_matrix.py
  6. 166
      main_package/classes/json_importer.py
  7. 198
      main_package/classes/network_graph.py
  8. 116
      main_package/classes/parameters_estimator.py
  9. 46
      main_package/classes/sample_path.py
  10. 75
      main_package/classes/set_of_cims.py
  11. 16
      main_package/classes/simple_cvs_importer.py
  12. 56
      main_package/classes/structure.py
  13. 159
      main_package/classes/structure_estimator.py
  14. 39
      main_package/classes/trajectory.py
  15. 180
      main_package/tests/test_networkgraph.py
  16. 11
      main_package/tests/test_structure_estimator.py

9
main_package/basic_main.py
Unescape View File

@ -1,9 +1,10 @@
import glob
import os
import sys import sys
sys.path.append("./classes/") sys.path.append("./classes/")
import glob
import os
import network_graph as ng import network_graph as ng
import sample_path as sp import sample_path as sp
import parameters_estimator as pe import parameters_estimator as pe
@ -24,7 +25,7 @@ def main():
#From The Structure Object build the Graph #From The Structure Object build the Graph
g = ng.NetworkGraph(s1.structure) g = ng.NetworkGraph(s1.structure)
#Select a node you want to estimate the parameters #Select a node you want to estimate the parameters
node = g.nodes[1] node = g.nodes[2]
print("NOde", node) print("NOde", node)
#Init the _graph specifically for THIS node #Init the _graph specifically for THIS node
g.fast_init(node) g.fast_init(node)

129
main_package/classes/abstract_importer.py
Unescape View File

@ -1,25 +1,24 @@
import sys
sys.path.append("./classes/")
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import pandas as pd import pandas as pd
import typing import typing
class AbstractImporter(ABC): class AbstractImporter(ABC):
""" """Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
Abstract class that exposes all the necessary methods to process the trajectories and the net structure.
:_file_path: the file path :param file_path: the file path
:_concatenated_samples: the concatenation of all the processed trajectories :type file_path: str
:_concatenated_samples: Dataframe containing the concatenation of all the processed trajectories
:_df_structure: Dataframe containing the structure of the network (edges) :_df_structure: Dataframe containing the structure of the network (edges)
:_df_variables: Dataframe containing the nodes cardinalities :_df_variables: Dataframe containing the nodes cardinalities
:_df_concatenated_samples: the concatenation and processing of all the trajectories present :_sorter: A list containing the columns header (excluding the time column) of the `_concatenated_samples`
in the list df_samples list
:_sorter: the columns header(excluding the time column) of the Dataframe concatenated_samples
""" """
def __init__(self, file_path: str): def __init__(self, file_path: str):
""" """Constructor
Parameters:
:_file_path: the path to the file containing the data
""" """
self._file_path = file_path self._file_path = file_path
self._df_variables = None self._df_variables = None
@ -29,46 +28,48 @@ class AbstractImporter(ABC):
super().__init__() super().__init__()
@abstractmethod @abstractmethod
def import_data(self): def import_data(self) -> None:
""" """Imports all the trajectories, variables cardinalities, and net edges.
Imports and prepares all data present needed for susequent computation.
Parameters: .. warning::
:void The class members ``_df_variables`` and ``_df_structure`` HAVE to be properly constructed
Returns: as Pandas Dataframes with the following structure:
:void Header of _df_structure = [From_Node | To_Node]
post[self]: the class members self._df_variables and self._df_structure HAVE to be properly constructed Header of _df_variables = [Variable_Label | Variable_Cardinality]
as Pandas Dataframes with the following structure: .. note::
Header of self._df_structure = [From_Node | To_Node] See :class:``JsonImporter`` for an example of implementation of this method.
Header of self.df_variables = [Variable_Label | Variable_Cardinality]
""" """
pass pass
@abstractmethod @abstractmethod
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List: def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
""" """Initializes the ``_sorter`` class member from a trajectory dataframe, exctracting the header of the frame
Initializes the self._sorter class member from a trajectory dataframe, exctracting the header of the frame
and keeping ONLY the variables symbolic labels, cutting out the time label in the header. and keeping ONLY the variables symbolic labels, cutting out the time label in the header.
Parameters:
:sample_frame: The dataframe from which extract the header :param sample_frame: The dataframe from which extract the header
Returns: :type sample_frame: pandas.DataFrame
:a list containing the processed header. :return: A list containing the processed header.
:rtype: List
""" """
pass pass
def compute_row_delta_sigle_samples_frame(self, sample_frame: pd.DataFrame, def compute_row_delta_sigle_samples_frame(self, sample_frame: pd.DataFrame,
columns_header: typing.List, shifted_cols_header: typing.List) \ columns_header: typing.List, shifted_cols_header: typing.List) \
-> pd.DataFrame: -> pd.DataFrame:
""" """Computes the difference between each value present in th time column.
Computes the difference between each value present in th time column.
Copies and shift by one position up all the values present in the remaining columns. Copies and shift by one position up all the values present in the remaining columns.
Parameters:
:sample_frame: the traj to be processed :param sample_frame: the traj to be processed
:time_header_label: the label for the times :type sample_frame: pandas.Dataframe
:columns_header: the original header of sample_frame :param columns_header: the original header of sample_frame
:shifted_cols_header: a copy of columns_header with changed names of the contents :type columns_header: List
Returns: :param shifted_cols_header: a copy of columns_header with changed names of the contents
:sample_frame: the processed dataframe :type shifted_cols_header: List
pre: the Dataframe sample_frame has to follow the column structure of this header: :return: The processed dataframe
:rtype: pandas.Dataframe
.. warning::
the Dataframe ``sample_frame`` has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values] Header of sample_frame = [Time | Variable values]
""" """
sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1) sample_frame.iloc[:, 0] = sample_frame.iloc[:, 0].diff().shift(-1)
@ -78,21 +79,23 @@ class AbstractImporter(ABC):
sample_frame.drop(sample_frame.tail(1).index, inplace=True) sample_frame.drop(sample_frame.tail(1).index, inplace=True)
return sample_frame return sample_frame
def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List): def compute_row_delta_in_all_samples_frames(self, df_samples_list: typing.List) -> None:
""" """Calls the method ``compute_row_delta_sigle_samples_frame`` on every dataframe present in the list
Calls the method compute_row_delta_sigle_samples_frame on every dataframe present in the list _df_samples_list. ``df_samples_list``.
Concatenates the result in the dataframe concatanated_samples Concatenates the result in the dataframe ``concatanated_samples``
Parameters:
time_header_label: the label of the time column :param df_samples_list: the datframe's list to be processed and concatenated
df_samples_list: the datframe's list to be processed and concatenated :type df_samples_list: List
Returns: .. warning::
void The Dataframe sample_frame has to follow the column structure of this header:
pre: the Dataframe sample_frame has to follow the column structure of this header:
Header of sample_frame = [Time | Variable values] Header of sample_frame = [Time | Variable values]
The class member self._sorter HAS to be properly INITIALIZED (See class members definition doc) The class member self._sorter HAS to be properly INITIALIZED (See class members definition doc)
.. note::
After the call of this method the class member ``concatanated_samples`` will contain all processed
and merged trajectories
""" """
if not self.sorter: if not self._sorter:
raise RuntimeError("The class member self._sorter has to be INITIALIZED!") raise RuntimeError("The class member self._sorter has to be INITIALIZED!")
shifted_cols_header = [s + "S" for s in self._sorter] shifted_cols_header = [s + "S" for s in self._sorter]
compute_row_delta = self.compute_row_delta_sigle_samples_frame compute_row_delta = self.compute_row_delta_sigle_samples_frame
@ -105,26 +108,28 @@ class AbstractImporter(ABC):
self._concatenated_samples = self._concatenated_samples[complete_header] self._concatenated_samples = self._concatenated_samples[complete_header]
def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List: def build_list_of_samples_array(self, data_frame: pd.DataFrame) -> typing.List:
""" """Builds a List containing the columns of data_frame and converts them to a numpy array.
Builds a List containing the columns of dataframe and converts them to a numpy array.
Parameters: :param data_frame: the dataframe from which the columns have to be extracted and converted
:data_frame: the dataframe from which the columns have to be extracted and converted :type data_frame: pandas.Dataframe
Returns: :return: the resulting list of numpy arrays
:columns_list: the resulting list of numpy arrays :rtype: List
""" """
columns_list = [data_frame[column].to_numpy() for column in data_frame] columns_list = [data_frame[column].to_numpy() for column in data_frame]
return columns_list return columns_list
def clear_concatenated_frame(self): def clear_concatenated_frame(self) -> None:
""" """Removes all values in the dataframe concatenated_samples.
Removes all values in the dataframe concatenated_samples
Parameters:
:void
Returns:
:void
""" """
self._concatenated_samples = self._concatenated_samples.iloc[0:0] self._concatenated_samples = self._concatenated_samples.iloc[0:0]
@abstractmethod
def dataset_id(self) -> object:
"""If the original dataset contains multiple dataset, this method returns a unique id to identify the current
dataset
"""
pass
@property @property
def concatenated_samples(self) -> pd.DataFrame: def concatenated_samples(self) -> pd.DataFrame:
return self._concatenated_samples return self._concatenated_samples

3
main_package/classes/abstract_sample_path.py
Unescape View File

@ -1,7 +1,10 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import abstract_importer as ai import abstract_importer as ai
import sys
sys.path.append('.')
class AbstractSamplePath(ABC): class AbstractSamplePath(ABC):

54
main_package/classes/cache.py
Unescape View File

@ -1,62 +1,54 @@
import sys
sys.path.append("./classes/")
import typing import typing
import set_of_cims as sofc import set_of_cims as sofc
class Cache: class Cache:
""" """This class acts as a cache of ``SetOfCims`` objects for a node.
This class has the role of a cache for SetOfCIMS of a test node that have been already computed during the ctpc algorithm.
:_list_of_sets_of_parents: a list of Sets of the parents to which the cim in cache at SAME index is related :_list_of_sets_of_parents: a list of ``Sets`` objects of the parents to which the cim in cache at SAME
index is related
:_actual_cache: a list of setOfCims objects :_actual_cache: a list of setOfCims objects
""" """
def __init__(self): def __init__(self):
"""Constructor Method
"""
self._list_of_sets_of_parents = [] self._list_of_sets_of_parents = []
self._actual_cache = [] self._actual_cache = []
def find(self, parents_comb: typing.Set) -> sofc.SetOfCims: def find(self, parents_comb: typing.Set) -> sofc.SetOfCims:
""" """
Tries to find in cache given the symbolic parents combination parents_comb the SetOfCims related to that parents_comb. Tries to find in cache given the symbolic parents combination ``parents_comb`` the ``SetOfCims``
Parameters: related to that ``parents_comb``.
parents_comb: the parents related to that SetOfCims
Returns:
A SetOfCims object if the parents_comb index is found in _list_of_sets_of_parents.
None otherwise.
:param parents_comb: the parents related to that ``SetOfCims``
:type parents_comb: Set
:return: A ``SetOfCims`` object if the ``parents_comb`` index is found in ``_list_of_sets_of_parents``.
None otherwise.
:rtype: SetOfCims
""" """
try: try:
#print("Cache State:", self.list_of_sets_of_indxs)
#print("Look For:", parents_comb)
result = self._actual_cache[self._list_of_sets_of_parents.index(parents_comb)] result = self._actual_cache[self._list_of_sets_of_parents.index(parents_comb)]
print("CACHE HIT!!!!", parents_comb)
return result return result
except ValueError: except ValueError:
return None return None
def put(self, parents_comb: typing.Union[typing.Set, str], socim: sofc.SetOfCims): def put(self, parents_comb: typing.Set, socim: sofc.SetOfCims) -> None:
""" """Place in cache the ``SetOfCims`` object, and the related symbolic index ``parents_comb`` in
Place in cache the SetOfCims object, and the related sybolyc index parents_comb in _list_of_sets_of_parents ``_list_of_sets_of_parents``.
Parameters:
parents_comb: the symbolic set index
socim: the related SetOfCims object
Returns: :param parents_comb: the symbolic set index
void :type parents_comb: Set
:param socim: the related SetOfCims object
:type socim: SetOfCims
""" """
#print("Putting in _cache:", parents_comb)
self._list_of_sets_of_parents.append(parents_comb) self._list_of_sets_of_parents.append(parents_comb)
self._actual_cache.append(socim) self._actual_cache.append(socim)
def clear(self): def clear(self) -> None:
""" """Clear the contents both of ``_actual_cache`` and ``_list_of_sets_of_parents``.
Clear the contents of both caches.
Parameters:
void
Returns:
void
""" """
del self._list_of_sets_of_parents[:] del self._list_of_sets_of_parents[:]
del self._actual_cache[:] del self._actual_cache[:]

25
main_package/classes/conditional_intensity_matrix.py
Unescape View File

@ -2,32 +2,25 @@ import numpy as np
class ConditionalIntensityMatrix: class ConditionalIntensityMatrix:
""" """Abstracts the Conditional Intesity matrix of a node as aggregation of the state residence times vector
Abstracts the Conditional Intesity matrix of a node as aggregation of the state residence times vector
and state transition matrix and the actual CIM matrix. and state transition matrix and the actual CIM matrix.
:_state_residence_times: state residence times vector :param state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix :type state_residence_times: numpy.array
:param state_transition_matrix: the transitions count matrix
:type state_transition_matrix: numpy.ndArray
:_cim: the actual cim of the node :_cim: the actual cim of the node
""" """
def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array): def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array):
""" """Constructor Method
Parameters:
:_state_residence_times: state residence times vector
:_state_transition_matrix: the transitions count matrix
""" """
self._state_residence_times = state_residence_times self._state_residence_times = state_residence_times
self._state_transition_matrix = state_transition_matrix self._state_transition_matrix = state_transition_matrix
self._cim = self.state_transition_matrix.astype(np.float64) self._cim = self.state_transition_matrix.astype(np.float64)
def compute_cim_coefficients(self): def compute_cim_coefficients(self) -> None:
""" """Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x].
Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x] The class member ``_cim`` will contain the computed cim
Parameters:
void
Returns:
void
""" """
np.fill_diagonal(self._cim, self._cim.diagonal() * -1) np.fill_diagonal(self._cim, self._cim.diagonal() * -1)
self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T

166
main_package/classes/json_importer.py
Unescape View File

@ -1,4 +1,5 @@
import sys
sys.path.append("./classes/")
import json import json
import typing import typing
import pandas as pd import pandas as pd
@ -7,34 +8,33 @@ import abstract_importer as ai
class JsonImporter(ai.AbstractImporter): class JsonImporter(ai.AbstractImporter):
""" """Implements the abstracts methods of AbstractImporter and adds all the necessary methods to process and prepare the data in json ext.
Implements the abstracts methods of AbstractImporter and adds all the necessary methods to process and prepare the data in json ext.
with the following structure: with the following structure:
[0] [0]
|_ dyn.cims |_ dyn.cims
|_ dyn.str |_ dyn.str
|_ samples |_ samples
|_ variabels |_ variabels
:file_path: the path of the file that contains tha data to be imported :param file_path: the path of the file that contains tha data to be imported
:_samples_label: the reference key for the samples in the trajectories :type file_path: string
:_structure_label: the reference key for the structure of the network data :param samples_label: the reference key for the samples in the trajectories
:_variables_label: the reference key for the cardinalites of the nodes data :type samples_label: string
:_time_key: the key used to identify the timestamps in each trajectory :param structure_label: the reference key for the structure of the network data
:_variables_key: the key used to identify the names of the variables in the net :type structure_label: string
:_df_samples_list: a Dataframe list in which every df contains a trajectory :param variables_label: the reference key for the cardinalites of the nodes data
:type variables_label: string
:param time_key: the key used to identify the timestamps in each trajectory
:type time_key: string
:param variables_key: the key used to identify the names of the variables in the net
:type variables_key: string
:param array_indx: the index of the outer JsonArray to exctract the data from
:type array_indx: int
:_df_samples_list: a Dataframe list in which every dataframe contains a trajectory
""" """
def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str, def __init__(self, file_path: str, samples_label: str, structure_label: str, variables_label: str, time_key: str,
variables_key: str, array_indx: int): variables_key: str, array_indx: int):
""" """Constructor method
Parameters:
:file_path: the path of the file that contains tha data to be imported
:_samples_label: the reference key for the samples in the trajectories
:_structure_label: the reference key for the structure of the network data
:_variables_label: the reference key for the cardinalites of the nodes data
:_time_key: the key used to identify the timestamps in each trajectory
:_variables_key: the key used to identify the names of the variables in the net
:_array_indx: the index of the outer json array from which import all the data
""" """
self._samples_label = samples_label self._samples_label = samples_label
self._structure_label = structure_label self._structure_label = structure_label
@ -45,13 +45,8 @@ class JsonImporter(ai.AbstractImporter):
self._array_indx = array_indx self._array_indx = array_indx
super(JsonImporter, self).__init__(file_path) super(JsonImporter, self).__init__(file_path)
def import_data(self): def import_data(self) -> None:
""" """Implements the abstract method of :class:`AbstractImporter`
Imports and prepares all data present needed for subsequent processing.
Parameters:
:void
Returns:
_void
""" """
raw_data = self.read_json_file() raw_data = self.read_json_file()
self._df_samples_list = self.import_trajectories(raw_data) self._df_samples_list = self.import_trajectories(raw_data)
@ -62,77 +57,71 @@ class JsonImporter(ai.AbstractImporter):
self._df_variables = self.import_variables(raw_data) self._df_variables = self.import_variables(raw_data)
def import_trajectories(self, raw_data: typing.List) -> typing.List: def import_trajectories(self, raw_data: typing.List) -> typing.List:
""" """Imports the trajectories from the list of dicts ``raw_data``.
Imports the trajectories in the list of dicts raw_data.
Parameters: :param raw_data: List of Dicts
:raw_data: List of Dicts :type raw_data: List
Returns: :return: List of dataframes containing all the trajectories
:List of dataframes containing all the trajectories :rtype: List
""" """
return self.normalize_trajectories(raw_data, self._array_indx, self._samples_label) return self.normalize_trajectories(raw_data, self._array_indx, self._samples_label)
def import_structure(self, raw_data: typing.List) -> pd.DataFrame: def import_structure(self, raw_data: typing.List) -> pd.DataFrame:
""" """Imports in a dataframe the data in the list raw_data at the key ``_structure_label``
Imports in a dataframe the data in the list raw_data at the key _structure_label
Parameters: :param raw_data: List of Dicts
:raw_data: the data :type raw_data: List
Returns: :return: Dataframe containg the starting node a ending node of every arc of the network
:Daframe containg the starting node a ending node of every arc of the network :rtype: pandas.Dataframe
""" """
return self.one_level_normalizing(raw_data, self._array_indx, self._structure_label) return self.one_level_normalizing(raw_data, self._array_indx, self._structure_label)
def import_variables(self, raw_data: typing.List) -> pd.DataFrame: def import_variables(self, raw_data: typing.List) -> pd.DataFrame:
""" """Imports the data in ``raw_data`` at the key ``_variables_label``.
Imports the data in raw_data at the key _variables_label.
Sorts the row of the dataframe df_variables using the list sorter. :param raw_data: List of Dicts
:type raw_data: List
Parameters: :return: Datframe containg the variables simbolic labels and their cardinalities
:raw_data: the data :rtype: pandas.Dataframe
:sorter: the header of the dataset containing only variables symbolic labels
Returns:
:Datframe containg the variables simbolic labels and their cardinalities
""" """
return self.one_level_normalizing(raw_data, self._array_indx, self._variables_label) return self.one_level_normalizing(raw_data, self._array_indx, self._variables_label)
def read_json_file(self) -> typing.List: def read_json_file(self) -> typing.List:
""" """Reads the JSON file in the path self.filePath
Reads the JSON file in the path self.filePath
Parameters:
:void
Returns:
:data: the contents of the json file
:return: The contents of the json file
:rtype: List
""" """
with open(self._file_path) as f: with open(self._file_path) as f:
data = json.load(f) data = json.load(f)
return data return data
def one_level_normalizing(self, raw_data: typing.List, indx: int, key: str) -> pd.DataFrame: def one_level_normalizing(self, raw_data: typing.List, indx: int, key: str) -> pd.DataFrame:
""" """Extracts the one-level nested data in the list ``raw_data`` at the index ``indx`` at the key ``key``.
Extracts the one-level nested data in the list raw_data at the index indx at the key key
Parameters:
:raw_data: List of Dicts
:indx: The index of the array from which the data have to be extracted
:key: the key for the Dicts from which exctract data
Returns:
:a normalized dataframe:
:param raw_data: List of Dicts
:type raw_data: List
:param indx: The index of the array from which the data have to be extracted
:type indx: int
:param key: the key for the Dicts from which exctract data
:type key: string
:return: A normalized dataframe
:rtype: pandas.Datframe
""" """
return pd.DataFrame(raw_data[indx][key]) return pd.DataFrame(raw_data[indx][key])
def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str) -> typing.List: def normalize_trajectories(self, raw_data: typing.List, indx: int, trajectories_key: str) -> typing.List:
""" """
Extracts the traj in raw_data at the index index at the key trajectories key. Extracts the trajectories in ``raw_data`` at the index ``index`` at the key ``trajectories key``.
Parameters: :param raw_data: List of Dicts
:raw_data: the data :type raw_data: List
:indx: the index of the array from which extract data :param indx: The index of the array from which the data have to be extracted
:trajectories_key: the key of the trajectories objects :type indx: int
Returns: :param trajectories_key: the key of the trajectories objects
:A list of daframes containg the trajectories :type trajectories_key: string
:return: A list of daframes containg the trajectories
:rtype: List
""" """
dataframe = pd.DataFrame dataframe = pd.DataFrame
smps = raw_data[indx][trajectories_key] smps = raw_data[indx][trajectories_key]
@ -140,34 +129,33 @@ class JsonImporter(ai.AbstractImporter):
return df_samples_list return df_samples_list
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List: def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
""" """Implements the abstract method build_sorter of the :class:`AbstractImporter` for this dataset
Implements the abstract method build_sorter for this dataset
""" """
columns_header = list(sample_frame.columns.values) columns_header = list(sample_frame.columns.values)
columns_header.remove(self._time_key) columns_header.remove(self._time_key)
return columns_header return columns_header
def clear_data_frame_list(self): def clear_data_frame_list(self) -> None:
""" """Removes all values present in the dataframes in the list ``_df_samples_list``.
Removes all values present in the dataframes in the list _df_samples_list
Parameters:
:void
Returns:
:void
""" """
for indx in range(len(self._df_samples_list)): for indx in range(len(self._df_samples_list)):
self._df_samples_list[indx] = self._df_samples_list[indx].iloc[0:0] self._df_samples_list[indx] = self._df_samples_list[indx].iloc[0:0]
def dataset_id(self) -> object:
return self._array_indx
def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict: def import_sampled_cims(self, raw_data: typing.List, indx: int, cims_key: str) -> typing.Dict:
""" """Imports the synthetic CIMS in the dataset in a dictionary, using variables labels
Imports the synthetic CIMS in the dataset in a dictionary, using variables labels
as keys for the set of CIMS of a particular node. as keys for the set of CIMS of a particular node.
Parameters:
:raw_data: the data :param raw_data: List of Dicts
:indx: the json array index :type raw_data: List
:cims_key: the key where the json object cims are placed :param indx: The index of the array from which the data have to be extracted
Returns: :type indx: int
:a dictionary containing the sampled CIMS for all the variables in the net :param cims_key: the key where the json object cims are placed
:type cims_key: string
:return: a dictionary containing the sampled CIMS for all the variables in the net
:rtype: Dictionary
""" """
cims_for_all_vars = {} cims_for_all_vars = {}
for var in raw_data[indx][cims_key]: for var in raw_data[indx][cims_key]:
@ -179,5 +167,3 @@ class JsonImporter(ai.AbstractImporter):

198
main_package/classes/network_graph.py
Unescape View File

@ -1,62 +1,54 @@
import sys
sys.path.append('./classes/')
import typing import typing
import structure as st
import networkx as nx import networkx as nx
import numpy as np import numpy as np
import structure as st
class NetworkGraph: class NetworkGraph:
""" """Abstracts the infos contained in the Structure class in the form of a directed graph.
Abstracts the infos contained in the Structure class in the form of a directed _graph. Has the task of creating all the necessary filtering and indexing structures for parameters estimation
Has the task of creating all the necessary filtering structures for parameters estimation
:param graph_struct: the ``Structure`` object from which infos about the net will be extracted
:_graph_struct: the Structure object from which infos about the net will be extracted :type graph_struct: Structure
:_graph: directed _graph :_graph: directed graph
:nodes_labels: the symbolic names of the variables :_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos
:nodes_indexes: the indexes of the nodes about every parents of the node of which all the indexing and filtering structures will be constructed.
:nodes_values: the cardinalites of the nodes
:_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos about every parents of every
node in the net
:_fancy_indexing: the indexes of every parent of every node in the net
:_time_scalar_indexing_structure: the indexing structure for state res time estimation :_time_scalar_indexing_structure: the indexing structure for state res time estimation
:_transition_scalar_indexing_structure: the indexing structure for transition computation :_transition_scalar_indexing_structure: the indexing structure for transition computation
:_time_filtering: the columns filtering structure used in the computation of the state res times :_time_filtering: the columns filtering structure used in the computation of the state res times
:_transition_filtering: the columns filtering structure used in the computation of the transition :_transition_filtering: the columns filtering structure used in the computation of the transition
from one state to another from one state to another
:self._p_combs_structure: all the possible parents states combination for every node in the net :_p_combs_structure: all the possible parents states combination for the node of interest
""" """
def __init__(self, graph_struct: st.Structure): def __init__(self, graph_struct: st.Structure):
""" """Constructor Method
Parameters:
:graph_struct:the Structure object from which infos about the net will be extracted
""" """
self._graph_struct = graph_struct self._graph_struct = graph_struct
self._graph = nx.DiGraph() self._graph = nx.DiGraph()
self._nodes_indexes = self._graph_struct.nodes_indexes
self._nodes_labels = self._graph_struct.nodes_labels
self._nodes_values = self._graph_struct.nodes_values
self._aggregated_info_about_nodes_parents = None self._aggregated_info_about_nodes_parents = None
self._fancy_indexing = None
self._time_scalar_indexing_structure = None self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None self._transition_scalar_indexing_structure = None
self._time_filtering = None self._time_filtering = None
self._transition_filtering = None self._transition_filtering = None
self._p_combs_structure = None self._p_combs_structure = None
def fast_init(self, node_id: str): def fast_init(self, node_id: str) -> None:
""" """Initializes all the necessary structures for parameters estimation of the node identified by the label
Initializes all the necessary structures for parameters estimation of the node identified by the label node_id node_id
Parameters:
node_id: the label of the node :param node_id: the label of the node
Returns: :type node_id: string
void
""" """
self.add_nodes(self._nodes_labels) self.add_nodes(self._graph_struct.nodes_labels)
self.add_edges(self._graph_struct.edges) self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id) self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
self._fancy_indexing = self._aggregated_info_about_nodes_parents[1] p_indxs = self._aggregated_info_about_nodes_parents[1]
p_indxs = self._fancy_indexing
p_vals = self._aggregated_info_about_nodes_parents[2] p_vals = self._aggregated_info_about_nodes_parents[2]
self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id, self._time_scalar_indexing_structure = self.build_time_scalar_indexing_structure_for_a_node(node_id,
p_vals) p_vals)
@ -67,48 +59,39 @@ class NetworkGraph:
self._transition_filtering = self.build_transition_filtering_for_a_node(node_indx, p_indxs) self._transition_filtering = self.build_transition_filtering_for_a_node(node_indx, p_indxs)
self._p_combs_structure = self.build_p_comb_structure_for_a_node(p_vals) self._p_combs_structure = self.build_p_comb_structure_for_a_node(p_vals)
def add_nodes(self, list_of_nodes: typing.List): def add_nodes(self, list_of_nodes: typing.List) -> None:
""" """Adds the nodes to the ``_graph`` contained in the list of nodes ``list_of_nodes``.
Adds the nodes to the _graph contained in the list of nodes list_of_nodes.
Sets all the properties that identify a nodes (index, positional index, cardinality) Sets all the properties that identify a nodes (index, positional index, cardinality)
Parameters: :param list_of_nodes: the nodes to add to ``_graph``
list_of_nodes: the nodes to add to _graph :type list_of_nodes: List
Returns:
void
""" """
nodes_indxs = self._nodes_indexes nodes_indxs = self._graph_struct.nodes_indexes
nodes_vals = self._graph_struct.nodes_values nodes_vals = self._graph_struct.nodes_values
pos = 0 pos = 0
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals): for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos) self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1 pos += 1
def add_edges(self, list_of_edges: typing.List): def add_edges(self, list_of_edges: typing.List) -> None:
""" """Add the edges to the ``_graph`` contained in the list ``list_of_edges``.
Add the edges to the _graph contained in the list list_of_edges.
Parameters: :param list_of_edges: the list containing of tuples containing the edges
list_of_edges :type list_of_edges: List
Returns:
void
""" """
self._graph.add_edges_from(list_of_edges) self._graph.add_edges_from(list_of_edges)
def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple: def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple:
""" """Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
(parents_labels, parents_indexes, parents_cardinalities). (parents_labels, parents_indexes, parents_cardinalities).
N.B. The parent set is sorted using the list of sorted nodes nodes
Parameters:
node: the label of the node
Returns:
a tuple containing all the parent set infos
:param node: the label of the node
:type node: string
:return: a tuple containing all the parent set infos
:rtype: Tuple
""" """
parents = self.get_parents_by_id(node) parents = self.get_parents_by_id(node)
nodes = self._nodes_labels nodes = self._graph_struct.nodes_labels
d = {v: i for i, v in enumerate(nodes)} d = {v: i for i, v in enumerate(nodes)}
sorted_parents = sorted(parents, key=lambda v: d[v]) sorted_parents = sorted(parents, key=lambda v: d[v])
get_node_indx = self.get_node_indx get_node_indx = self.get_node_indx
@ -117,15 +100,14 @@ class NetworkGraph:
return (sorted_parents, p_indxes, p_values) return (sorted_parents, p_indxes, p_values)
def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray: def build_time_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) -> np.ndarray:
""" """Builds an indexing structure for the computation of state residence times values.
Builds an indexing structure for the computation of state residence times values.
:param node_id: the node label
Parameters: :type node_id: string
node_id: the node label :param parents_vals: the caridinalites of the node's parents
parents_vals: the caridinalites of the node's parents :type parents_vals: List
Returns: :return: The time indexing structure
a numpy array. :rtype: numpy.ndArray
""" """
T_vector = np.array([self.get_states_number(node_id)]) T_vector = np.array([self.get_states_number(node_id)])
T_vector = np.append(T_vector, parents_vals) T_vector = np.append(T_vector, parents_vals)
@ -134,15 +116,14 @@ class NetworkGraph:
def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) \ def build_transition_scalar_indexing_structure_for_a_node(self, node_id: str, parents_vals: typing.List) \
-> np.ndarray: -> np.ndarray:
""" """Builds an indexing structure for the computation of state transitions values.
Builds an indexing structure for the computation of state transitions values.
:param node_id: the node label
Parameters: :type node_id: string
node_id: the node label :param parents_vals: the caridinalites of the node's parents
parents_vals: the caridinalites of the node's parents :type parents_vals: List
Returns: :return: The transition indexing structure
a numpy array. :rtype: numpy.ndArray
""" """
node_states_number = self.get_states_number(node_id) node_states_number = self.get_states_number(node_id)
M_vector = np.array([node_states_number, M_vector = np.array([node_states_number,
@ -153,37 +134,41 @@ class NetworkGraph:
def build_time_columns_filtering_for_a_node(self, node_indx: int, p_indxs: typing.List) -> np.ndarray: def build_time_columns_filtering_for_a_node(self, node_indx: int, p_indxs: typing.List) -> np.ndarray:
""" """
Builds the necessary structure to filter the desired columns indicated by node_indx and p_indxs in the dataset. Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
in the dataset.
This structute will be used in the computation of the state res times. This structute will be used in the computation of the state res times.
Parameters: :param node_indx: the index of the node
node_indx: the index of the node :type node_indx: int
p_indxs: the indexes of the node's parents :param p_indxs: the indexes of the node's parents
Returns: :type p_indxs: List
a numpy array :return: The filtering structure for times estimation
:rtype: numpy.ndArray
""" """
return np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int) return np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
def build_transition_filtering_for_a_node(self, node_indx, p_indxs) -> np.ndarray: def build_transition_filtering_for_a_node(self, node_indx: int, p_indxs: typing.List) -> np.ndarray:
""" """Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
Builds the necessary structure to filter the desired columns indicated by node_indx and p_indxs in the dataset. in the dataset.
This structute will be used in the computation of the state transitions values. This structure will be used in the computation of the state transitions values.
Parameters: :param node_indx: the index of the node
node_indx: the index of the node :type node_indx: int
p_indxs: the indexes of the node's parents :param p_indxs: the indexes of the node's parents
Returns: :type p_indxs: List
a numpy array :return: The filtering structure for transitions estimation
:rtype: numpy.ndArray
""" """
nodes_number = self._graph_struct.total_variables_number nodes_number = self._graph_struct.total_variables_number
return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int) return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray: def build_p_comb_structure_for_a_node(self, parents_values: typing.List) -> np.ndarray:
""" """
Builds the combinatory structure that contains the combinations of all the values contained in parents_values. Builds the combinatorial structure that contains the combinations of all the values contained in
``parents_values``.
Parameters: :param parents_values: the cardinalities of the nodes
parents_values: the cardinalities of the nodes :type parents_values: List
Returns: :return: A numpy matrix containing a grid of the combinations
a numpy matrix containinga grid of the combinations :rtype: numpy.ndArray
""" """
tmp = [] tmp = []
for val in parents_values: for val in parents_values:
@ -198,21 +183,28 @@ class NetworkGraph:
parents_comb = np.array([[]], dtype=np.int) parents_comb = np.array([[]], dtype=np.int)
return parents_comb return parents_comb
def get_parents_by_id(self, node_id): def get_parents_by_id(self, node_id) -> typing.List:
"""Returns a list of labels of the parents of the node ``node_id``
:param node_id: the node label
:type node_id: string
:return: a List of labels of the parents
:rtype: List
"""
return list(self._graph.predecessors(node_id)) return list(self._graph.predecessors(node_id))
def get_states_number(self, node_id): def get_states_number(self, node_id) -> int:
return self._graph.nodes[node_id]['val'] return self._graph.nodes[node_id]['val']
def get_node_indx(self, node_id): def get_node_indx(self, node_id) -> int:
return nx.get_node_attributes(self._graph, 'indx')[node_id] return nx.get_node_attributes(self._graph, 'indx')[node_id]
def get_positional_node_indx(self, node_id): def get_positional_node_indx(self, node_id) -> int:
return self._graph.nodes[node_id]['pos_indx'] return self._graph.nodes[node_id]['pos_indx']
@property @property
def nodes(self) -> typing.List: def nodes(self) -> typing.List:
return self._nodes_labels return self._graph_struct.nodes_labels
@property @property
def edges(self) -> typing.List: def edges(self) -> typing.List:
@ -220,11 +212,11 @@ class NetworkGraph:
@property @property
def nodes_indexes(self) -> np.ndarray: def nodes_indexes(self) -> np.ndarray:
return self._nodes_indexes return self._graph_struct.nodes_indexes
@property @property
def nodes_values(self) -> np.ndarray: def nodes_values(self) -> np.ndarray:
return self._nodes_values return self._graph_struct.nodes_values
@property @property
def time_scalar_indexing_strucure(self) -> np.ndarray: def time_scalar_indexing_strucure(self) -> np.ndarray:
@ -246,7 +238,8 @@ class NetworkGraph:
def p_combs(self) -> np.ndarray: def p_combs(self) -> np.ndarray:
return self._p_combs_structure return self._p_combs_structure
"""##############These Methods are actually unused but could become useful in the near future################""" """
##############These Methods are actually unused but could become useful in the near future################
def init_graph(self): def init_graph(self):
self.add_nodes(self._nodes_labels) self.add_nodes(self._nodes_labels)
@ -302,4 +295,5 @@ class NetworkGraph:
pass pass
else: else:
fancy_indx = [i[1] for i in self._aggregated_info_about_nodes_parents] fancy_indx = [i[1] for i in self._aggregated_info_about_nodes_parents]
return fancy_indx return fancy_indx
"""

116
main_package/classes/parameters_estimator.py
Unescape View File

@ -1,60 +1,52 @@
import sys
sys.path.append("./classes/")
import numpy as np import numpy as np
import network_graph as ng import network_graph as ng
import trajectory as tr import trajectory as tr
import set_of_cims as sofc import set_of_cims as sofc
import sets_of_cims_container as acims
class ParametersEstimator: class ParametersEstimator:
""" """Has the task of computing the cims of particular node given the trajectories and the net structure
Has the task of computing the cims of particular node given the trajectories in samplepath and the net structure in the graph ``_net_graph``.
in the graph _net_graph
:trajectories: the trajectories :param trajectories: the trajectories
:_net_graph: the net structure :type trajectories: Trajectory
:param net_graph: the net structure
:type net_graph: NetworkGraph
:_single_set_of_cims: the set of cims object that will hold the cims of the node :_single_set_of_cims: the set of cims object that will hold the cims of the node
""" """
def __init__(self, trajectories: tr.Trajectory, net_graph: ng.NetworkGraph): def __init__(self, trajectories: tr.Trajectory, net_graph: ng.NetworkGraph):
"""Constructor Method
""" """
Parameters:
:trajectories: the trajectories
:_net_graph: the net structure
"""
#self.sample_path = sample_path
self._trajectories = trajectories self._trajectories = trajectories
self._net_graph = net_graph self._net_graph = net_graph
#self.sets_of_cims_struct = None
self._single_set_of_cims = None self._single_set_of_cims = None
def fast_init(self, node_id: str): def fast_init(self, node_id: str) -> None:
""" """Initializes all the necessary structures for the parameters estimation for the node ``node_id``.
Initializes all the necessary structures for the parameters estimation.
Parameters: :param node_id: the node label
node_id: the node label :type node_id: string
Returns:
void
""" """
p_vals = self._net_graph._aggregated_info_about_nodes_parents[2] p_vals = self._net_graph._aggregated_info_about_nodes_parents[2]
node_states_number = self._net_graph.get_states_number(node_id) node_states_number = self._net_graph.get_states_number(node_id)
self._single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs) self._single_set_of_cims = sofc.SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs)
def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims: def compute_parameters_for_node(self, node_id: str) -> sofc.SetOfCims:
""" """Compute the CIMS of the node identified by the label ``node_id``.
Compute the CIMS of the node identified by the label node_id
Parameters: :param node_id: the node label
node_id: the node label :type node_id: string
Returns: :return: A SetOfCims object filled with the computed CIMS
A setOfCims object filled with the computed CIMS :rtype: SetOfCims
""" """
node_indx = self._net_graph.get_node_indx(node_id) node_indx = self._net_graph.get_node_indx(node_id)
state_res_times = self._single_set_of_cims._state_residence_times state_res_times = self._single_set_of_cims._state_residence_times
transition_matrices = self._single_set_of_cims._transition_matrices transition_matrices = self._single_set_of_cims._transition_matrices
#trajectory = self.sample_path.trajectories.trajectory
self.compute_state_res_time_for_node(node_indx, self._trajectories.times, self.compute_state_res_time_for_node(node_indx, self._trajectories.times,
self._trajectories.trajectory, self._trajectories.trajectory,
self._net_graph.time_filtering, self._net_graph.time_filtering,
@ -69,59 +61,54 @@ class ParametersEstimator:
return self._single_set_of_cims return self._single_set_of_cims
def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray, def compute_state_res_time_for_node(self, node_indx: int, times: np.ndarray, trajectory: np.ndarray,
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray, T: np.ndarray): cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray,
""" T: np.ndarray) -> None:
Compute the state residence times for a node and fill the matrix T with the results """Compute the state residence times for a node and fill the matrix ``T`` with the results
Parameters: :param node_indx: the index of the node
node_indx: the index of the node :type node_indx: int
times: the times deltas vector :param times: the times deltas vector
trajectory: the trajectory :type times: numpy.array
cols_filter: the columns filtering structure :param trajectory: the trajectory
scalar_indexes_struct: the indexing structure :type trajectory: numpy.ndArray
T: the state residence times vectors :param cols_filter: the columns filtering structure
Returns: :type cols_filter: numpy.array
void :param scalar_indexes_struct: the indexing structure
:type scalar_indexes_struct: numpy.array
:param T: the state residence times vectors
:type T: numpy.ndArray
""" """
T[:] = np.bincount(np.sum(trajectory[:, cols_filter] * scalar_indexes_struct / scalar_indexes_struct[0], axis=1) T[:] = np.bincount(np.sum(trajectory[:, cols_filter] * scalar_indexes_struct / scalar_indexes_struct[0], axis=1)
.astype(np.int), \ .astype(np.int), \
times, times,
minlength=scalar_indexes_struct[-1]).reshape(-1, T.shape[1]) minlength=scalar_indexes_struct[-1]).reshape(-1, T.shape[1])
def compute_state_transitions_for_a_node(self, node_indx, trajectory, cols_filter, scalar_indexing, M): def compute_state_transitions_for_a_node(self, node_indx: int, trajectory: np.ndarray, cols_filter: np.ndarray,
""" scalar_indexing: np.ndarray, M: np.ndarray):
Compute the state residence times for a node and fill the matrices M with the results """Compute the state residence times for a node and fill the matrices ``M`` with the results.
Parameters: :param node_indx: the index of the node
node_indx: the index of the node :type node_indx: int
times: the times deltas vector :param trajectory: the trajectory
trajectory: the trajectory :type trajectory: numpy.ndArray
cols_filter: the columns filtering structure :param cols_filter: the columns filtering structure
scalar_indexes: the indexing structure :type cols_filter: numpy.array
M: the state transition matrices :param scalar_indexing: the indexing structure
Returns: :type scalar_indexing: numpy.array
void :param M: the state transitions matrices
:type M: numpy.ndArray
""" """
diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])], diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])],
dtype=np.int64) dtype=np.int64)
trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0] trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0]
#print("Trajectory", trajectory)
#print("Step 1", trajectory[:, int(trajectory.shape[1] / 2) + node_indx])
#print("Step 2", trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0)
#print("TrTemp", trj_tmp)
#print("Cols Filter", cols_filter)
#print("Filtered Tr Temp", trj_tmp[:, cols_filter])
#print("Actual Indexing", scalar_indexing / scalar_indexing[0])
#print("PreBins",trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0] )
#print("Bins", np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1))
#print("After BinCount", np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int)))
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int), M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int),
minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2]) minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
M_raveled = M.ravel() M_raveled = M.ravel()
M_raveled[diag_indices] = 0 M_raveled[diag_indices] = 0
M_raveled[diag_indices] = np.sum(M, axis=2).ravel() M_raveled[diag_indices] = np.sum(M, axis=2).ravel()
"""##############These Methods are actually unused but could become useful in the near future################""" """
##############These Methods are actually unused but could become useful in the near future################
def init_sets_cims_container(self): def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self._net_graph.nodes, self.sets_of_cims_struct = acims.SetsOfCimsContainer(self._net_graph.nodes,
@ -143,6 +130,7 @@ class ParametersEstimator:
self._net_graph.transition_scalar_indexing_structure[indx], self._net_graph.transition_scalar_indexing_structure[indx],
aggr[1]._transition_matrices) aggr[1]._transition_matrices)
aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices) aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices)
"""

46
main_package/classes/sample_path.py
Unescape View File

@ -3,23 +3,23 @@ import abstract_importer as imp
import structure as st import structure as st
import trajectory as tr import trajectory as tr
import sys
sys.path.append('./classes/')
class SamplePath:
"""
Aggregates all the informations about the trajectories, the real structure of the sampled net and variables
cardinalites.
Has the task of creating the objects that will contain the mentioned data.
:_importer: the Importer objects that will import ad process data class SamplePath:
:trajectories: the Trajectory object that will contain all the concatenated trajectories """Aggregates all the informations about the trajectories, the real structure of the sampled net and variables
:structure: the Structure Object that will contain all the structurral infos about the net cardinalites. Has the task of creating the objects ``Trajectory`` and ``Structure`` that will
contain the mentioned data.
:param importer: the Importer objects that will import ad process data
:type importer: AbstractImporter
:_trajectories: the ``Trajectory`` object that will contain all the concatenated trajectories
:_structure: the ``Structure`` Object that will contain all the structurral infos about the net
:_total_variables_count: the number of variables in the net :_total_variables_count: the number of variables in the net
""" """
def __init__(self, importer: imp.AbstractImporter): def __init__(self, importer: imp.AbstractImporter):
""" """Constructor Method
Parameters:
:importer: the Importer objects that will import ad process data
""" """
self._importer = importer self._importer = importer
self._trajectories = None self._trajectories = None
@ -27,30 +27,18 @@ class SamplePath:
self._total_variables_count = None self._total_variables_count = None
self._importer.import_data() self._importer.import_data()
def build_trajectories(self): def build_trajectories(self) -> None:
""" """Builds the Trajectory object that will contain all the trajectories.
Builds the Trajectory object that will contain all the trajectories. Clears all the unused dataframes in ``_importer`` Object
Clears all the unused dataframes in Importer Object
Parameters:
:void
Returns:
:void
""" """
#self._importer.import_data()
self._trajectories = \ self._trajectories = \
tr.Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples), tr.Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples),
len(self._importer.sorter) + 1) len(self._importer.sorter) + 1)
#self.trajectories.append(trajectory)
self._importer.clear_concatenated_frame() self._importer.clear_concatenated_frame()
def build_structure(self): def build_structure(self) -> None:
""" """
Builds the Structure object that aggregates all the infos about the net. Builds the ``Structure`` object that aggregates all the infos about the net.
Parameters:
:void
Returns:
:void
""" """
if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list(): if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list():
raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!") raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!")

75
main_package/classes/set_of_cims.py
Unescape View File

@ -1,31 +1,29 @@
import typing import typing
import numpy as np import numpy as np
import sys
sys.path.append("./classes/")
import conditional_intensity_matrix as cim import conditional_intensity_matrix as cim
class SetOfCims: class SetOfCims:
""" """Aggregates all the CIMS of the node identified by the label _node_id.
Aggregates all the CIMS of the node identified by the label _node_id.
:param node_id: the node label
:_node_id: the node label :type node_ind: string
:_parents_states_number: the cardinalities of the parents :param parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node :type parents_states_number: List
:_p_combs: the relative p_comb structure :param node_states_number: the caridinality of the node
:state_residence_time: matrix containing all the state residence time vectors for the node :type node_states_number: int
:param p_combs: the p_comb structure bound to this node
:type p_combs: numpy.ndArray
:_state_residence_time: matrix containing all the state residence time vectors for the node
:_transition_matrices: matrix containing all the transition matrices for the node :_transition_matrices: matrix containing all the transition matrices for the node
:_actual_cims: the cims of the node :_actual_cims: the cims of the node
""" """
def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray): def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray):
""" """Constructor Method
Parameters:
:_node_id: the node label
:_parents_states_number: the cardinalities of the parents
:_node_states_number: the caridinality of the node
:_p_combs: the relative p_comb structure
""" """
self._node_id = node_id self._node_id = node_id
self._parents_states_number = parents_states_number self._parents_states_number = parents_states_number
@ -36,14 +34,8 @@ class SetOfCims:
self._p_combs = p_combs self._p_combs = p_combs
self.build_times_and_transitions_structures() self.build_times_and_transitions_structures()
def build_times_and_transitions_structures(self): def build_times_and_transitions_structures(self) -> None:
""" """Initializes at the correct dimensions the state residence times matrix and the state transition matrices.
Initializes at the correct dimensions the state residence times matrix and the state transition matrices
Parameters:
:void
Returns:
:void
""" """
if not self._parents_states_number: if not self._parents_states_number:
self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float) self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float)
@ -54,16 +46,14 @@ class SetOfCims:
self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number, self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number,
self._node_states_number], dtype=np.int) self._node_states_number], dtype=np.int)
def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray): def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray) -> None:
""" """Build the ``ConditionalIntensityMatrix`` objects given the state residence times and transitions matrices.
Build the ConditionalIntensityMatrix object given the state residence times and transitions matrices. Compute the cim coefficients.The class member ``_actual_cims`` will contain the computed cims.
Compute the cim coefficients.
:param state_res_times: the state residence times matrix
Parameters: :type state_res_times: numpy.ndArray
:state_res_times: the state residence times matrix :param transition_matrices: the transition matrices
:_transition_matrices: the transition matrices :type transition_matrices: numpy.ndArray
Returns:
:void
""" """
for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices): for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices):
cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix) cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix)
@ -74,14 +64,15 @@ class SetOfCims:
self._state_residence_times = None self._state_residence_times = None
def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray: def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray:
""" """Filter the cims contained in the array ``_actual_cims`` given the boolean mask ``mask_arr`` and the index
Filter the cims contained in the array _actual_cims given the boolean mask mask_arr and the index comb. ``comb``.
Parameters:
:mask_arr: the boolean mask :param mask_arr: the boolean mask that indicates which parent to consider
:comb: the indexes of the selected cims :type mask_arr: numpy.array
:param comb: the state/s of the filtered parents
Returns: :type comb: numpy.array
:Array of ConditionalIntensityMatrix :return: Array of ``ConditionalIntensityMatrix`` objects
:rtype: numpy.array
""" """
if mask_arr.size <= 1: if mask_arr.size <= 1:
return self._actual_cims return self._actual_cims

16
main_package/classes/simple_cvs_importer.py
Unescape View File

@ -2,6 +2,8 @@ import pandas as pd
import glob import glob
import os import os
import typing
import abstract_importer as ai import abstract_importer as ai
import sample_path as sp import sample_path as sp
@ -14,6 +16,7 @@ class CSVImporter(ai.AbstractImporter):
def import_data(self): def import_data(self):
self.read_csv_file() self.read_csv_file()
self._sorter = self.build_sorter(self._df_samples_list[0])
self.import_variables() self.import_variables()
self.import_structure() self.import_structure()
self.compute_row_delta_in_all_samples_frames(self._df_samples_list) self.compute_row_delta_in_all_samples_frames(self._df_samples_list)
@ -24,16 +27,15 @@ class CSVImporter(ai.AbstractImporter):
self._df_samples_list = [df] self._df_samples_list = [df]
def import_variables(self): def import_variables(self):
variables_list = list(self._df_samples_list[0].columns)[1:] values_list = [3 for var in self._sorter]
#wrong_vars_labels = ['Y','Z','X'] # initialize dict of lists
self._sorter = variables_list data = {'Name':self._sorter, 'Value':values_list}
values_list = [3 for var in variables_list]
# initialize list of lists
data = {'Name':variables_list, 'Value':values_list}
# Create the pandas DataFrame # Create the pandas DataFrame
self._df_variables = pd.DataFrame(data) self._df_variables = pd.DataFrame(data)
def build_sorter(self, sample_frame: pd.DataFrame) -> typing.List:
return list(sample_frame.columns)[1:]
def import_structure(self): def import_structure(self):
data = {'From':['X','Y','Z'], 'To':['Z','Z','Y']} data = {'From':['X','Y','Z'], 'To':['Z','Z','Y']}
self._df_structure = pd.DataFrame(data) self._df_structure = pd.DataFrame(data)

56
main_package/classes/structure.py
Unescape View File

@ -4,29 +4,26 @@ import numpy as np
class Structure: class Structure:
"""Contains all the infos about the network structure(nodes labels, nodes caridinalites, edges, indexes)
:param nodes_labels_list: the symbolic names of the variables
:type nodes_labels_list: List
:param nodes_indexes_arr: the indexes of the nodes
:type nodes_indexes_arr: numpy.ndArray
:param nodes_vals_arr: the cardinalites of the nodes
:type nodes_vals_arr: numpy.ndArray
:param edges_list: the edges of the network
:type edges_list: List
:param total_variables_number: the total number of variables in the net
:type total_variables_number: int
""" """
Contains all the infos about the network structure(nodes names, nodes caridinalites, edges...)
:nodes_labels_list: the symbolic names of the variables def __init__(self, nodes_labels_list: ty.List, nodes_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
def __init__(self, nodes_label_list: ty.List, node_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
edges_list: ty.List, total_variables_number: int): edges_list: ty.List, total_variables_number: int):
"""Constructor Method
""" """
Parameters: self._nodes_labels_list = nodes_labels_list
:nodes_labels_list: the symbolic names of the variables self._nodes_indexes_arr = nodes_indexes_arr
:nodes_indexes_arr: the indexes of the nodes
:nodes_vals_arr: the cardinalites of the nodes
:edges_list: the edges of the network
:total_variables_number: the total number of variables in the net
"""
self._nodes_labels_list = nodes_label_list
self._nodes_indexes_arr = node_indexes_arr
self._nodes_vals_arr = nodes_vals_arr self._nodes_vals_arr = nodes_vals_arr
self._edges_list = edges_list self._edges_list = edges_list
self._total_variables_number = total_variables_number self._total_variables_number = total_variables_number
@ -52,9 +49,23 @@ class Structure:
return self._total_variables_number return self._total_variables_number
def get_node_id(self, node_indx: int) -> str: def get_node_id(self, node_indx: int) -> str:
"""Given the ``node_index`` returns the node label.
:param node_indx: the node index
:type node_indx: int
:return: the node label
:rtype: string
"""
return self._nodes_labels_list[node_indx] return self._nodes_labels_list[node_indx]
def get_node_indx(self, node_id: str) -> int: def get_node_indx(self, node_id: str) -> int:
"""Given the ``node_index`` returns the node label.
:param node_id: the node label
:type node_id: string
:return: the node index
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node_id) pos_indx = self._nodes_labels_list.index(node_id)
return self._nodes_indexes_arr[pos_indx] return self._nodes_indexes_arr[pos_indx]
@ -62,6 +73,13 @@ class Structure:
return self._nodes_labels_list.index(node_id) return self._nodes_labels_list.index(node_id)
def get_states_number(self, node: str) -> int: def get_states_number(self, node: str) -> int:
"""Given the node label ``node`` returns the cardinality of the node.
:param node: the node label
:type node: string
:return: the node cardinality
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node) pos_indx = self._nodes_labels_list.index(node)
return self._nodes_vals_arr[pos_indx] return self._nodes_vals_arr[pos_indx]

159
main_package/classes/structure_estimator.py
Unescape View File

@ -1,8 +1,11 @@
import sys
sys.path.append("./classes/")
from tqdm import tqdm
import itertools import itertools
import json import json
import typing import typing
import networkx as nx import networkx as nx
import numpy as np import numpy as np
from networkx.readwrite import json_graph from networkx.readwrite import json_graph
@ -18,27 +21,23 @@ import structure as st
class StructureEstimator: class StructureEstimator:
""" """Has the task of estimating the network structure given the trajectories in ``samplepath``.
Has the task of estimating the network structure given the trajectories in samplepath.
:param sample_path: the _sample_path object containing the trajectories and the real structure
:_sample_path: the _sample_path object containing the trajectories and the real structure :type sample_path: SamplePath
:_exp_test_sign: the significance level for the exponential Hp test :param exp_test_alfa: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test :type exp_test_alfa: float
:param chi_test_alfa: the significance level for the chi Hp test
:_nodes: the _nodes labels :type chi_test_alfa: float
:_nodes: the nodes labels
:_nodes_vals: the nodes cardinalities :_nodes_vals: the nodes cardinalities
:_nodes_indxs: the nodes indexes :_nodes_indxs: the nodes indexes
:_complete_graph: the complete directed graph built using the nodes labels in nodes :_complete_graph: the complete directed graph built using the nodes labels in ``_nodes``
:_cache: the _cache object :_cache: the Cache object
""" """
def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float): def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float):
""" """Constructor Method
Parameters:
:_sample_path: the _sample_path object containing the trajectories and the real structure
:_exp_test_sign: the significance level for the exponential Hp test
:_chi_test_alfa: the significance level for the chi Hp test
""" """
self._sample_path = sample_path self._sample_path = sample_path
self._nodes = np.array(self._sample_path.structure.nodes_labels) self._nodes = np.array(self._sample_path.structure.nodes_labels)
@ -50,13 +49,12 @@ class StructureEstimator:
self._cache = ch.Cache() self._cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List) -> nx.DiGraph: def build_complete_graph(self, node_ids: typing.List) -> nx.DiGraph:
""" """Builds a complete directed graph (no self loops) given the nodes labels in the list ``node_ids``:
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
Parameters: :param node_ids: the list of nodes labels
node_ids: the list of nodes labels :type node_ids: List
Returns: :return: a complete Digraph Object
a complete Digraph Object :rtype: networkx.DiGraph
""" """
complete_graph = nx.DiGraph() complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids) complete_graph.add_nodes_from(node_ids)
@ -65,20 +63,22 @@ class StructureEstimator:
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int, def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int) -> bool: tot_vars_count: int) -> bool:
""" """Performs a complete independence test on the directed graphs G1 = {test_child U parent_set}
Performs a complete independence test on the directed graphs G1 = test_child U parent_set G2 = {G1 U test_parent} (added as an additional parent of the test_child).
G2 = G1 U test_parent (added as an additional parent of the test_child).
Generates all the necessary structures and datas to perform the tests. Generates all the necessary structures and datas to perform the tests.
Parameters: :param test_parent: the node label of the test parent
test_parent: the node label of the test parent :type test_parent: string
test_child: the node label of the child :param test_child: the node label of the child
parent_set: the common parent set :type test_child: string
child_states_numb: the cardinality of the test_child :param parent_set: the common parent set
tot_vars_count: the total number of variables in the net :type parent_set: List
Returns: :param child_states_numb: the cardinality of the ``test_child``
True iff test_child and test_parent are independent given the sep_set parent_set :type child_states_numb: int
False otherwise :param tot_vars_count: the total number of variables in the net
:type tot_vars_count: int
:return: True iff test_child and test_parent are independent given the sep_set parent_set. False otherwise
:rtype: bool
""" """
p_set = parent_set[:] p_set = parent_set[:]
complete_info = parent_set[:] complete_info = parent_set[:]
@ -130,19 +130,18 @@ class StructureEstimator:
def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix, def independence_test(self, child_states_numb: int, cim1: condim.ConditionalIntensityMatrix,
cim2: condim.ConditionalIntensityMatrix) -> bool: cim2: condim.ConditionalIntensityMatrix) -> bool:
""" """Compute the actual independence test using two cims.
Compute the actual independence test using two cims.
It is performed first the exponential test and if the null hypothesis is not rejected, It is performed first the exponential test and if the null hypothesis is not rejected,
it is permormed also the chi_test. it is performed also the chi_test.
Parameters:
child_states_numb: the cardinality of the test child
cim1: a cim belonging to the graph without test parent
cim2: a cim belonging to the graph with test parent
Returns: :param child_states_numb: the cardinality of the test child
True iff both tests do NOT reject the null hypothesis of indipendence :type child_states_numb: int
False otherwise :param cim1: a cim belonging to the graph without test parent
:type cim1: ConditionalIntensityMatrix
:param cim2: a cim belonging to the graph with test parent
:type cim2: ConditionalIntensityMatrix
:return:True iff both tests do NOT reject the null hypothesis of indipendence. False otherwise.
:rtype: bool
""" """
M1 = cim1.state_transition_matrix M1 = cim1.state_transition_matrix
M2 = cim2.state_transition_matrix M2 = cim2.state_transition_matrix
@ -155,7 +154,6 @@ class StructureEstimator:
for val in range(0, child_states_numb): for val in range(0, child_states_numb):
if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \ if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \
F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]): F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]):
#print("CONDITIONALLY DEPENDENT EXP")
return False return False
M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1) M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1)
M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape( M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape(
@ -170,17 +168,15 @@ class StructureEstimator:
return False return False
return True return True
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int): def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int) -> None:
""" """Performs an iteration of the CTPC algorithm using the node ``var_id`` as ``test_child``.
Performs an iteration of the CTPC algorithm using the node var_id as test_child.
Parameters: :param var_id: the node label of the test child
var_id: the node label of the test child :type var_id: string
tot_vars_count: the number of _nodes in the net :param tot_vars_count: the number of _nodes in the net
Returns: :type tot_vars_count: int
void
""" """
print("##################TESTING VAR################", var_id) #print("##################TESTING VAR################", var_id)
u = list(self._complete_graph.predecessors(var_id)) u = list(self._complete_graph.predecessors(var_id))
child_states_numb = self._sample_path.structure.get_states_number(var_id) child_states_numb = self._sample_path.structure.get_states_number(var_id)
b = 0 b = 0
@ -190,12 +186,8 @@ class StructureEstimator:
removed = False removed = False
S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx]) S = self.generate_possible_sub_sets_of_size(u, b, u[parent_indx])
test_parent = u[parent_indx] test_parent = u[parent_indx]
#print("Test Parent", test_parent)
for parents_set in S: for parents_set in S:
print("Parent Set", parents_set)
print("Test Parent", test_parent)
if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count): if self.complete_test(test_parent, var_id, parents_set, child_states_numb, tot_vars_count):
#print("Removing EDGE:", test_parent, var_id)
self._complete_graph.remove_edge(test_parent, var_id) self._complete_graph.remove_edge(test_parent, var_id)
u.remove(test_parent) u.remove(test_parent)
removed = True removed = True
@ -207,46 +199,37 @@ class StructureEstimator:
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str) -> \ def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str) -> \
typing.Iterator: typing.Iterator:
""" """Creates a list containing all possible subsets of the list ``u`` of size ``size``,
Creates a list containing all possible subsets of the list u of size size, that do not contains a the node identified by ``parent_label``.
that do not contains a the node identified by parent_label.
:param u: the list of nodes
Parameters: :type u: List
u: the list of _nodes :param size: the size of the subsets
size: the size of the subsets :type size: int
parent_label: the _nodes to exclude in the subsets generation :param parent_label: the node to exclude in the subsets generation
Returns: :type parent_label: string
a Map Object containing a list of lists :return: an Iterator Object containing a list of lists
:rtype: Iterator
""" """
list_without_test_parent = u[:] list_without_test_parent = u[:]
list_without_test_parent.remove(parent_label) list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size)) return map(list, itertools.combinations(list_without_test_parent, size))
def ctpc_algorithm(self): def ctpc_algorithm(self) -> None:
""" """Compute the CTPC algorithm over the entire net.
Compute the CTPC algorithm.
Parameters:
void
Returns:
void
""" """
ctpc_algo = self.one_iteration_of_CTPC_algorithm ctpc_algo = self.one_iteration_of_CTPC_algorithm
total_vars_numb = self._sample_path.total_variables_count total_vars_numb = self._sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self._nodes] [ctpc_algo(n, total_vars_numb) for n in tqdm(self._nodes)]
def save_results(self): def save_results(self) -> None:
""" """Save the estimated Structure to a .json file in the path where the data are loaded from.
Save the estimated Structure to a .json file The file is named as the input dataset but the results_ word is appendend to the results file.
Parameters:
void
Returns:
void
""" """
res = json_graph.node_link_data(self._complete_graph) res = json_graph.node_link_data(self._complete_graph)
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1] name = self._sample_path._importer.file_path.rsplit('/', 1)[-1] + str(self._sample_path._importer.dataset_id())
name = 'results_' + name #TODO va aggiunto anche l'indice di array name = 'results_' + name
with open(name, 'w') as f: with open(name, 'w') as f:
json.dump(res, f) json.dump(res, f)

39
main_package/classes/trajectory.py
Unescape View File

@ -1,25 +1,22 @@
import numpy as np import numpy as np
import typing
class Trajectory: class Trajectory:
""" """ Abstracts the infos about a complete set of trajectories, represented as a numpy array of doubles (the time deltas)
Abstracts the infos about a complete set of trajectories, represented as a numpy array of doubles and a numpy matrix and a numpy matrix of ints (the changes of states).
of ints.
:param list_of_columns: the list containing the times array and values matrix
:list_of_columns: the list containing the times array and values matrix :type list_of_columns: List
:original_cols_numb: total number of cols in the data :param original_cols_number: total number of cols in the data
:actual_trajectory: the trajectory containing also the duplicated and shifted values :type original_cols_number: int
:times: the array containing the time deltas :_actual_trajectory: the trajectory containing also the duplicated/shifted values
:_times: the array containing the time deltas
""" """
def __init__(self, list_of_columns, original_cols_number): def __init__(self, list_of_columns: typing.List, original_cols_number: int):
""" """Constructor Method
Parameters:
:list_of_columns: the list containing the times array and values matrix
:original_cols_numb: total number of cols in the data
""" """
if type(list_of_columns[0][0]) != np.float64: if type(list_of_columns[0][0]) != np.float64:
raise TypeError('The first array in the list has to be Times') raise TypeError('The first array in the list has to be Times')
@ -29,22 +26,10 @@ class Trajectory:
@property @property
def trajectory(self) -> np.ndarray: def trajectory(self) -> np.ndarray:
"""
Parameters:
void
Returns:
a numpy matrix containing ONLY the original columns values, not the shifted ones
"""
return self._actual_trajectory[:, :self._original_cols_number] return self._actual_trajectory[:, :self._original_cols_number]
@property @property
def complete_trajectory(self) -> np.ndarray: def complete_trajectory(self) -> np.ndarray:
"""
Parameters:
void
Returns:
a numpy matrix containing all the values
"""
return self._actual_trajectory return self._actual_trajectory
@property @property

180
main_package/tests/test_networkgraph.py
Unescape View File

@ -25,10 +25,10 @@ class TestNetworkGraph(unittest.TestCase):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
self.assertEqual(self.s1.structure, g1._graph_struct) self.assertEqual(self.s1.structure, g1._graph_struct)
self.assertIsInstance(g1._graph, nx.DiGraph) self.assertIsInstance(g1._graph, nx.DiGraph)
self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes)) #self.assertTrue(np.array_equal(g1._nodes_indexes, self.s1.structure.nodes_indexes))
self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels) #self.assertListEqual(g1._nodes_labels, self.s1.structure.nodes_labels)
self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values)) #self.assertTrue(np.array_equal(g1._nodes_values, self.s1.structure.nodes_values))
self.assertIsNone(g1._fancy_indexing) #self.assertIsNone(g1._fancy_indexing)
self.assertIsNone(g1.time_scalar_indexing_strucure) self.assertIsNone(g1.time_scalar_indexing_strucure)
self.assertIsNone(g1.transition_scalar_indexing_structure) self.assertIsNone(g1.transition_scalar_indexing_structure)
self.assertIsNone(g1.transition_filtering) self.assertIsNone(g1.transition_filtering)
@ -46,28 +46,40 @@ class TestNetworkGraph(unittest.TestCase):
for e in self.s1.structure.edges: for e in self.s1.structure.edges:
self.assertIn(tuple(e), g1.edges) self.assertIn(tuple(e), g1.edges)
def aux_aggregated_par_list_data(self, graph, node_id, sorted_par_list_aggregated_info): def test_fast_init(self):
for indx, element in enumerate(sorted_par_list_aggregated_info): g1 = ng.NetworkGraph(self.s1.structure)
if indx == 0: for node in self.s1.structure.nodes_labels:
self.assertEqual(graph.get_parents_by_id(node_id), element) g1.fast_init(node)
for j in range(0, len(sorted_par_list_aggregated_info[0]) - 1): self.assertIsNotNone(g1._graph.nodes)
self.assertLess(self.s1.structure.get_node_indx(sorted_par_list_aggregated_info[0][j]), self.assertIsNotNone(g1._graph.edges)
self.s1.structure.get_node_indx(sorted_par_list_aggregated_info[0][j + 1])) self.assertIsInstance(g1._time_scalar_indexing_structure, np.ndarray)
elif indx == 1: self.assertIsInstance(g1._transition_scalar_indexing_structure, np.ndarray)
for node, node_indx in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[1]): self.assertIsInstance(g1._time_filtering, np.ndarray)
self.assertEqual(graph.get_node_indx(node), node_indx) self.assertIsInstance(g1._transition_filtering, np.ndarray)
else: self.assertIsInstance(g1._p_combs_structure, np.ndarray)
for node, node_val in zip(sorted_par_list_aggregated_info[0], sorted_par_list_aggregated_info[2]): self.assertIsInstance(g1._aggregated_info_about_nodes_parents, tuple)
self.assertEqual(graph._graph_struct.get_states_number(node), node_val)
def test_get_ordered_by_indx_set_of_parents(self):
def test_get_ord_set_of_par_of_all_nodes(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
sorted_list_of_par_lists = g1.get_ord_set_of_par_of_all_nodes() for node in self.s1.structure.nodes_labels:
for node, par_list in zip(g1.nodes, sorted_list_of_par_lists): aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_aggregated_par_list_data(g1, node, par_list) for indx in range(len(aggr_info[0]) - 1 ):
self.assertLess(g1.get_node_indx(aggr_info[0][indx]), g1.get_node_indx(aggr_info[0][indx + 1]))
for par, par_indx in zip(aggr_info[0], aggr_info[1]):
self.assertEqual(g1.get_node_indx(par), par_indx)
for par, par_val in zip(aggr_info[0], aggr_info[2]):
self.assertEqual(g1._graph_struct.get_states_number(par), par_val)
def test_build_time_scalar_indexing_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node, aggr_info[1],
aggr_info[0], aggr_info[2])
def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals): def aux_build_time_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, parents_vals):
time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals) time_scalar_indexing = graph.build_time_scalar_indexing_structure_for_a_node(node_id, parents_vals)
@ -81,6 +93,15 @@ class TestNetworkGraph(unittest.TestCase):
t_vec = t_vec.cumprod() t_vec = t_vec.cumprod()
self.assertTrue(np.array_equal(time_scalar_indexing, t_vec)) self.assertTrue(np.array_equal(time_scalar_indexing, t_vec))
def test_build_transition_scalar_indexing_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
for node in self.s1.structure.nodes_labels:
aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node, aggr_info[1],
aggr_info[0], aggr_info[2])
def aux_build_transition_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels, def aux_build_transition_scalar_indexing_structure_for_a_node(self, graph, node_id, parents_indxs, parents_labels,
parents_values): parents_values):
transition_scalar_indexing = graph.build_transition_scalar_indexing_structure_for_a_node(node_id, transition_scalar_indexing = graph.build_transition_scalar_indexing_structure_for_a_node(node_id,
@ -96,93 +117,54 @@ class TestNetworkGraph(unittest.TestCase):
m_vec = m_vec.cumprod() m_vec = m_vec.cumprod()
self.assertTrue(np.array_equal(transition_scalar_indexing, m_vec)) self.assertTrue(np.array_equal(transition_scalar_indexing, m_vec))
def test_build_transition_scalar_indexing_structure(self): def test_build_time_columns_filtering_structure_for_a_node(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() for node in self.s1.structure.nodes_labels:
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents] aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes() self.aux_build_time_columns_filtering_structure_for_a_node(g1, node, aggr_info[1])
fancy_indx = g1.build_fancy_indexing_structure(0)
for node_id, p_i ,p_l, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals): def aux_build_time_columns_filtering_structure_for_a_node(self, graph, node_id, p_indxs):
self.aux_build_transition_scalar_indexing_structure_for_a_node(g1, node_id, p_i ,p_l, p_v) graph.build_time_columns_filtering_for_a_node(graph.get_node_indx(node_id), p_indxs)
single_filter = []
def test_build_time_scalar_indexing_structure(self): single_filter.append(graph.get_node_indx(node_id))
g1 = ng.NetworkGraph(self.s1.structure) single_filter.extend(p_indxs)
g1.add_nodes(self.s1.structure.nodes_labels) self.assertTrue(np.array_equal(graph.build_time_columns_filtering_for_a_node(graph.get_node_indx(node_id),
g1.add_edges(self.s1.structure.edges) p_indxs),np.array(single_filter)))
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
fancy_indx = g1.build_fancy_indexing_structure(0)
p_labels = [i[0] for i in g1._aggregated_info_about_nodes_parents]
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes()
#print(fancy_indx)
for node_id, p_indxs, p_labels, p_v in zip(g1._graph_struct.nodes_labels, fancy_indx, p_labels, p_vals):
self.aux_build_time_scalar_indexing_structure_for_a_node(g1, node_id, p_indxs, p_labels, p_v)
def test_build_time_columns_filtering_structure(self):
g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes()
g1._fancy_indexing = g1.build_fancy_indexing_structure(0)
g1.build_time_columns_filtering_structure()
t_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing):
single_filter = []
single_filter.append(g1.get_node_indx(node_id))
single_filter.extend(p_indxs)
t_filter.append(np.array(single_filter))
#print(t_filter)
for a1, a2 in zip(g1.time_filtering, t_filter):
self.assertTrue(np.array_equal(a1, a2))
def test_build_transition_columns_filtering_structure(self): def test_build_transition_columns_filtering_structure(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() for node in self.s1.structure.nodes_labels:
g1._fancy_indexing = g1.build_fancy_indexing_structure(0) aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
g1.build_transition_columns_filtering_structure() self.aux_build_time_columns_filtering_structure_for_a_node(g1, node, aggr_info[1])
m_filter = []
for node_id, p_indxs in zip(g1.nodes, g1._fancy_indexing): def aux_build_transition_columns_filtering_structure(self, graph, node_id, p_indxs):
single_filter = [] single_filter = []
single_filter.append(g1.get_node_indx(node_id) + g1._graph_struct.total_variables_number) single_filter.append(graph.get_node_indx(node_id) + graph._graph_struct.total_variables_number)
single_filter.append(g1.get_node_indx(node_id)) single_filter.append(graph.get_node_indx(node_id))
single_filter.extend(p_indxs) single_filter.extend(p_indxs)
m_filter.append(np.array(single_filter)) self.assertTrue(np.array_equal(graph.build_transition_filtering_for_a_node(graph.get_node_indx(node_id),
for a1, a2 in zip(g1.transition_filtering, m_filter):
self.assertTrue(np.array_equal(a1, a2)) p_indxs), np.array(single_filter)))
def test_build_p_combs_structure(self): def test_build_p_combs_structure(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)
g1.add_nodes(self.s1.structure.nodes_labels) g1.add_nodes(self.s1.structure.nodes_labels)
g1.add_edges(self.s1.structure.edges) g1.add_edges(self.s1.structure.edges)
g1._aggregated_info_about_nodes_parents = g1.get_ord_set_of_par_of_all_nodes() for node in self.s1.structure.nodes_labels:
p_vals = g1.get_ordered_by_indx_parents_values_for_all_nodes() aggr_info = g1.get_ordered_by_indx_set_of_parents(node)
p_combs = g1.build_p_combs_structure() self.aux_build_p_combs_structure(g1, aggr_info[2])
for matrix, p_v in zip(p_combs, p_vals): def aux_build_p_combs_structure(self, graph, p_vals):
p_possible_vals = [] p_combs = graph.build_p_comb_structure_for_a_node(p_vals)
for val in p_v: p_possible_vals = []
vals = [v for v in range(val)] for val in p_vals:
p_possible_vals.extend(vals) vals = [v for v in range(val)]
comb_struct = set(itertools.product(p_possible_vals,repeat=len(p_v))) p_possible_vals.extend(vals)
#print(comb_struct) comb_struct = set(itertools.product(p_possible_vals,repeat=len(p_vals)))
for comb in comb_struct: for comb in comb_struct:
self.assertIn(np.array(comb), matrix) self.assertIn(np.array(comb), p_combs)
def test_fast_init(self):
g1 = ng.NetworkGraph(self.s1.structure)
g2 = ng.NetworkGraph(self.s1.structure)
g1.init_graph()
for indx, node in enumerate(g1.nodes):
g2.fast_init(node)
self.assertListEqual(g2._fancy_indexing, g1._fancy_indexing[indx])
self.assertTrue(np.array_equal(g2.time_scalar_indexing_strucure, g1.time_scalar_indexing_strucure[indx]))
self.assertTrue(np.array_equal(g2.transition_scalar_indexing_structure, g1.transition_scalar_indexing_structure[indx]))
self.assertTrue(np.array_equal(g2.time_filtering, g1.time_filtering[indx]))
self.assertTrue(np.array_equal(g2.transition_filtering, g1.transition_filtering[indx]))
self.assertTrue(np.array_equal(g2.p_combs, g1.p_combs[indx]))
def test_get_parents_by_id(self): def test_get_parents_by_id(self):
g1 = ng.NetworkGraph(self.s1.structure) g1 = ng.NetworkGraph(self.s1.structure)

11
main_package/tests/test_structure_estimator.py
Unescape View File

@ -9,6 +9,7 @@ import networkx as nx
import numpy as np import numpy as np
import psutil import psutil
from line_profiler import LineProfiler from line_profiler import LineProfiler
import timeit
import cache as ch import cache as ch
import sample_path as sp import sample_path as sp
@ -21,7 +22,7 @@ class TestStructureEstimator(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json")) cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 3) cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name', 0)
cls.s1 = sp.SamplePath(cls.importer) cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories() cls.s1.build_trajectories()
cls.s1.build_structure() cls.s1.build_structure()
@ -70,12 +71,14 @@ class TestStructureEstimator(unittest.TestCase):
def test_time(self): def test_time(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1) se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
lp = LineProfiler() lp = LineProfiler()
lp.add_function(se1.complete_test) #lp.add_function(se1.complete_test)
lp.add_function(se1.one_iteration_of_CTPC_algorithm) #lp.add_function(se1.one_iteration_of_CTPC_algorithm)
lp.add_function(se1.independence_test) #lp.add_function(se1.independence_test)
lp_wrapper = lp(se1.ctpc_algorithm) lp_wrapper = lp(se1.ctpc_algorithm)
lp_wrapper() lp_wrapper()
lp.print_stats() lp.print_stats()
#print("Last time", lp.dump_stats())
#print("Exec Time", timeit.timeit(se1.ctpc_algorithm, number=1))
print(se1._complete_graph.edges) print(se1._complete_graph.edges)
print(self.s1.structure.edges) print(self.s1.structure.edges)
for ed in self.s1.structure.edges: for ed in self.s1.structure.edges: