1
0
Fork 0

Cleaned code

master
Luca Moretti 4 years ago
parent c53a65ebd7
commit a3e614bcb8
  1. 143
      main_package/classes/estimators/parameters_estimator.py.bak
  2. 245
      main_package/classes/estimators/structure_constraint_based_estimator.py.bak
  3. 189
      main_package/classes/estimators/structure_estimator.py.bak
  4. 2
      main_package/classes/estimators/structure_score_based_estimator.py
  5. 41
      main_package/classes/structure_graph/abstract_sample_path.py
  6. 44
      main_package/classes/structure_graph/conditional_intensity_matrix.py.bak
  7. 285
      main_package/classes/structure_graph/network_graph.py.bak
  8. 95
      main_package/classes/structure_graph/sample_path.py.bak
  9. 98
      main_package/classes/structure_graph/set_of_cims.py.bak
  10. 28
      main_package/classes/structure_graph/sets_of_cims_container.py
  11. 128
      main_package/classes/structure_graph/structure.py.bak
  12. 28
      main_package/classes/utility/decorators.py
  13. 1729
      main_package/tests/results/results copy.csv
  14. 14
      main_package/tests/results/results.csv
  15. 20
      main_package/tests/results/results16-32RAM.csv
  16. 1153
      main_package/tests/results/results_tabu.csv
  17. 147
      main_package/tests/simulators/test_simulation_all.py
  18. 108
      main_package/tests/simulators/test_simulation_constraint.py
  19. 122
      main_package/tests/simulators/test_simulation_score.py

@ -1,143 +0,0 @@
import sys
sys.path.append('../')
import numpy as np
from ..structure_graph.network_graph import NetworkGraph
from ..structure_graph.sample_path import SetOfCims
from ..structure_graph.trajectory import Trajectory
class ParametersEstimator(object):
"""Has the task of computing the cims of particular node given the trajectories and the net structure
in the graph ``_net_graph``.
:param trajectories: the trajectories
:type trajectories: Trajectory
:param net_graph: the net structure
:type net_graph: NetworkGraph
:_single_set_of_cims: the set of cims object that will hold the cims of the node
"""
def __init__(self, trajectories: Trajectory, net_graph: NetworkGraph):
"""Constructor Method
"""
self._trajectories = trajectories
self._net_graph = net_graph
self._single_set_of_cims = None
def fast_init(self, node_id: str) -> None:
"""Initializes all the necessary structures for the parameters estimation for the node ``node_id``.
:param node_id: the node label
:type node_id: string
"""
p_vals = self._net_graph._aggregated_info_about_nodes_parents[2]
node_states_number = self._net_graph.get_states_number(node_id)
self._single_set_of_cims = SetOfCims(node_id, p_vals, node_states_number, self._net_graph.p_combs)
def compute_parameters_for_node(self, node_id: str) -> SetOfCims:
"""Compute the CIMS of the node identified by the label ``node_id``.
:param node_id: the node label
:type node_id: string
:return: A SetOfCims object filled with the computed CIMS
:rtype: SetOfCims
"""
node_indx = self._net_graph.get_node_indx(node_id)
state_res_times = self._single_set_of_cims._state_residence_times
transition_matrices = self._single_set_of_cims._transition_matrices
ParametersEstimator.compute_state_res_time_for_node(self._trajectories.times,
self._trajectories.trajectory,
self._net_graph.time_filtering,
self._net_graph.time_scalar_indexing_strucure,
state_res_times)
ParametersEstimator.compute_state_transitions_for_a_node(node_indx, self._trajectories.complete_trajectory,
self._net_graph.transition_filtering,
self._net_graph.transition_scalar_indexing_structure,
transition_matrices)
self._single_set_of_cims.build_cims(state_res_times, transition_matrices)
return self._single_set_of_cims
@staticmethod
def compute_state_res_time_for_node(times: np.ndarray, trajectory: np.ndarray,
cols_filter: np.ndarray, scalar_indexes_struct: np.ndarray,
T: np.ndarray) -> None:
"""Compute the state residence times for a node and fill the matrix ``T`` with the results
:param node_indx: the index of the node
:type node_indx: int
:param times: the times deltas vector
:type times: numpy.array
:param trajectory: the trajectory
:type trajectory: numpy.ndArray
:param cols_filter: the columns filtering structure
:type cols_filter: numpy.array
:param scalar_indexes_struct: the indexing structure
:type scalar_indexes_struct: numpy.array
:param T: the state residence times vectors
:type T: numpy.ndArray
"""
T[:] = np.bincount(np.sum(trajectory[:, cols_filter] * scalar_indexes_struct / scalar_indexes_struct[0], axis=1)
.astype(np.int), \
times,
minlength=scalar_indexes_struct[-1]).reshape(-1, T.shape[1])
@staticmethod
def compute_state_transitions_for_a_node(node_indx: int, trajectory: np.ndarray, cols_filter: np.ndarray,
scalar_indexing: np.ndarray, M: np.ndarray) -> None:
"""Compute the state residence times for a node and fill the matrices ``M`` with the results.
:param node_indx: the index of the node
:type node_indx: int
:param trajectory: the trajectory
:type trajectory: numpy.ndArray
:param cols_filter: the columns filtering structure
:type cols_filter: numpy.array
:param scalar_indexing: the indexing structure
:type scalar_indexing: numpy.array
:param M: the state transitions matrices
:type M: numpy.ndArray
"""
diag_indices = np.array([x * M.shape[1] + x % M.shape[1] for x in range(M.shape[0] * M.shape[1])],
dtype=np.int64)
trj_tmp = trajectory[trajectory[:, int(trajectory.shape[1] / 2) + node_indx].astype(np.int) >= 0]
M[:] = np.bincount(np.sum(trj_tmp[:, cols_filter] * scalar_indexing / scalar_indexing[0], axis=1).astype(np.int)
, minlength=scalar_indexing[-1]).reshape(-1, M.shape[1], M.shape[2])
M_raveled = M.ravel()
M_raveled[diag_indices] = 0
M_raveled[diag_indices] = np.sum(M, axis=2).ravel()
def init_sets_cims_container(self):
self.sets_of_cims_struct = acims.SetsOfCimsContainer(self.net_graph.nodes,
self.net_graph.nodes_values,
self.net_graph.get_ordered_by_indx_parents_values_for_all_nodes(),
self.net_graph.p_combs)
def compute_parameters(self):
#print(self.net_graph.get_nodes())
#print(self.amalgamated_cims_struct.sets_of_cims)
#enumerate(zip(self.net_graph.get_nodes(), self.amalgamated_cims_struct.sets_of_cims))
for indx, aggr in enumerate(zip(self.net_graph.nodes, self.sets_of_cims_struct.sets_of_cims)):
#print(self.net_graph.time_filtering[indx])
#print(self.net_graph.time_scalar_indexing_strucure[indx])
self.compute_state_res_time_for_node(self.net_graph.get_node_indx(aggr[0]), self.sample_path.trajectories.times,
self.sample_path.trajectories.trajectory,
self.net_graph.time_filtering[indx],
self.net_graph.time_scalar_indexing_strucure[indx],
aggr[1]._state_residence_times)
#print(self.net_graph.transition_filtering[indx])
#print(self.net_graph.transition_scalar_indexing_structure[indx])
self.compute_state_transitions_for_a_node(self.net_graph.get_node_indx(aggr[0]),
self.sample_path.trajectories.complete_trajectory,
self.net_graph.transition_filtering[indx],
self.net_graph.transition_scalar_indexing_structure[indx],
aggr[1]._transition_matrices)
aggr[1].build_cims(aggr[1]._state_residence_times, aggr[1]._transition_matrices)

@ -1,245 +0,0 @@
import sys
sys.path.append('../')
import itertools
import json
import typing
import networkx as nx
import numpy as np
from networkx.readwrite import json_graph
import os
from scipy.stats import chi2 as chi2_dist
from scipy.stats import f as f_dist
from tqdm import tqdm
from ..utility.cache as ch
from ..structure_graph.conditional_intensity_matrix import ConditionalIntensityMatrix
from ..structure_graph.network_graph import NetworkGraph
from .parameters_estimator import ParametersEstimator
from .structure_estimator import StructureEstimator
from ..structure_graph.sample_path import SamplePath
from ..structure_graph.structure import Structure
from ..optimizers.constraint_based_optimizer import ConstraintBasedOptimizer
import concurrent.futures
from utility.decorators import timing,timing_write
import multiprocessing
from multiprocessing import Pool
class StructureConstraintBasedEstimator(se.StructureEstimator):
"""
Has the task of estimating the network structure given the trajectories in samplepath by using a constraint-based approach.
:param sample_path: the _sample_path object containing the trajectories and the real structure
:type sample_path: SamplePath
:param exp_test_alfa: the significance level for the exponential Hp test
:type exp_test_alfa: float
:param chi_test_alfa: the significance level for the chi Hp test
:type chi_test_alfa: float
:_nodes: the nodes labels
:_nodes_vals: the nodes cardinalities
:_nodes_indxs: the nodes indexes
:_complete_graph: the complete directed graph built using the nodes labels in ``_nodes``
:_cache: the Cache object
"""
def __init__(self, sample_path: SamplePath, exp_test_alfa: float, chi_test_alfa: float,known_edges: typing.List= [],thumb_threshold:int = 25):
super().__init__(sample_path,known_edges)
self._exp_test_sign = exp_test_alfa
self._chi_test_alfa = chi_test_alfa
self._thumb_threshold = thumb_threshold
tot_vars_count: int, parent_indx, child_indx) -> bool:
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int, parent_indx, child_indx) -> bool:
"""Performs a complete independence test on the directed graphs G1 = {test_child U parent_set}
G2 = {G1 U test_parent} (added as an additional parent of the test_child).
Generates all the necessary structures and datas to perform the tests.
:param test_parent: the node label of the test parent
:type test_parent: string
:param test_child: the node label of the child
:type test_child: string
:param parent_set: the common parent set
:type parent_set: List
:param child_states_numb: the cardinality of the ``test_child``
:type child_states_numb: int
:param tot_vars_count: the total number of variables in the net
:type tot_vars_count: int
:return: True iff test_child and test_parent are independent given the sep_set parent_set. False otherwise
:rtype: bool
"""
p_set = parent_set[:]
complete_info = parent_set[:]
complete_info.append(test_child)
parents = np.array(parent_set)
parents = np.append(parents, test_parent)
sorted_parents = self._nodes[np.isin(self._nodes, parents)]
cims_filter = sorted_parents != test_parent
p_set.insert(0, test_parent)
sofc2 = self._cache.find(set(p_set))
if not sofc2:
complete_info.append(test_parent)
bool_mask2 = np.isin(self._nodes, complete_info)
l2 = list(self._nodes[bool_mask2])
indxs2 = self._nodes_indxs[bool_mask2]
vals2 = self._nodes_vals[bool_mask2]
eds2 = list(itertools.product(p_set, test_child))
s2 = Structure(l2, indxs2, vals2, eds2, tot_vars_count)
g2 = NetworkGraph(s2)
g2.fast_init(test_child)
p2 = ParametersEstimator(self._sample_path.trajectories, g2)
p2.fast_init(test_child)
sofc2 = p2.compute_parameters_for_node(test_child)
self._cache.put(set(p_set), sofc2)
del p_set[0]
sofc1 = self._cache.find(set(p_set))
if not sofc1:
g2.remove_node(test_parent)
g2.fast_init(test_child)
p2 = ParametersEstimator(self._sample_path.trajectories, g2)
p2.fast_init(test_child)
sofc1 = p2.compute_parameters_for_node(test_child)
self._cache.put(set(p_set), sofc1)
thumb_value = 0.0
if child_states_numb > 2:
parent_val = self._sample_path.structure.get_states_number(test_parent)
bool_mask_vals = np.isin(self._nodes, parent_set)
parents_vals = self._nodes_vals[bool_mask_vals]
thumb_value = self.compute_thumb_value(parent_val, child_states_numb, parents_vals)
for cim1, p_comb in zip(sofc1.actual_cims, sofc1.p_combs):
cond_cims = sofc2.filter_cims_with_mask(cims_filter, p_comb)
for cim2 in cond_cims:
if not self.independence_test(child_states_numb, cim1, cim2, thumb_value, parent_indx, child_indx):
return False
return True
def independence_test(self, child_states_numb: int, cim1: ConditionalIntensityMatrix,
cim2: ConditionalIntensityMatrix, thumb_value: float, parent_indx, child_indx) -> bool:
"""Compute the actual independence test using two cims.
It is performed first the exponential test and if the null hypothesis is not rejected,
it is performed also the chi_test.
:param child_states_numb: the cardinality of the test child
:type child_states_numb: int
:param cim1: a cim belonging to the graph without test parent
:type cim1: ConditionalIntensityMatrix
:param cim2: a cim belonging to the graph with test parent
:type cim2: ConditionalIntensityMatrix
:return: True iff both tests do NOT reject the null hypothesis of independence. False otherwise.
:rtype: bool
"""
M1 = cim1.state_transition_matrix
M2 = cim2.state_transition_matrix
r1s = M1.diagonal()
r2s = M2.diagonal()
C1 = cim1.cim
C2 = cim2.cim
if child_states_numb > 2:
if (np.sum(np.diagonal(M1)) / thumb_value) < self._thumb_threshold:
self._removable_edges_matrix[parent_indx][child_indx] = False
return False
F_stats = C2.diagonal() / C1.diagonal()
exp_alfa = self._exp_test_sign
for val in range(0, child_states_numb):
if F_stats[val] < f_dist.ppf(exp_alfa / 2, r1s[val], r2s[val]) or \
F_stats[val] > f_dist.ppf(1 - exp_alfa / 2, r1s[val], r2s[val]):
return False
M1_no_diag = M1[~np.eye(M1.shape[0], dtype=bool)].reshape(M1.shape[0], -1)
M2_no_diag = M2[~np.eye(M2.shape[0], dtype=bool)].reshape(
M2.shape[0], -1)
chi_2_quantile = chi2_dist.ppf(1 - self._chi_test_alfa, child_states_numb - 1)
Ks = np.sqrt(r1s / r2s)
Ls = np.sqrt(r2s / r1s)
for val in range(0, child_states_numb):
Chi = np.sum(np.power(Ks[val] * M2_no_diag[val] - Ls[val] *M1_no_diag[val], 2) /
(M1_no_diag[val] + M2_no_diag[val]))
if Chi > chi_2_quantile:
return False
return True
def compute_thumb_value(self, parent_val, child_val, parent_set_vals):
"""Compute the value to test against the thumb_threshold.
:param parent_val: test parent's variable cardinality
:type parent_val: int
:param child_val: test child's variable cardinality
:type child_val: int
:param parent_set_vals: the cardinalities of the nodes in the current sep-set
:type parent_set_vals: List
:return: the thumb value for the current independence test
:rtype: int
"""
df = (child_val - 1) ** 2
df = df * parent_val
for v in parent_set_vals:
df = df * v
return df
def one_iteration_of_CTPC_algorithm(self, var_id: str, tot_vars_count: int)-> typing.List:
"""Performs an iteration of the CTPC algorithm using the node ``var_id`` as ``test_child``.
:param var_id: the node label of the test child
:type var_id: string
"""
optimizer_obj = optimizer.ConstraintBasedOptimizer(
node_id = var_id,
structure_estimator = self,
tot_vars_count = tot_vars_count)
return optimizer_obj.optimize_structure()
def ctpc_algorithm(self,disable_multiprocessing:bool= False ):
"""
Compute the CTPC algorithm.
Parameters:
void
Returns:
void
"""
ctpc_algo = self.one_iteration_of_CTPC_algorithm
total_vars_numb = self._sample_path.total_variables_count
n_nodes= len(self.nodes)
total_vars_numb_array = [total_vars_numb] * n_nodes
'get the number of CPU'
cpu_count = multiprocessing.cpu_count()
'Remove all the edges from the structure'
self._sample_path.structure.clean_structure_edges()
'Estimate the best parents for each node'
#with multiprocessing.Pool(processes=cpu_count) as pool:
#with get_context("spawn").Pool(processes=cpu_count) as pool:
if disable_multiprocessing:
print("DISABILITATO")
cpu_count = 1
list_edges_partial = [ctpc_algo(n,total_vars_numb) for n in self.nodes]
else:
with concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count) as executor:
list_edges_partial = executor.map(ctpc_algo,
self.nodes,
total_vars_numb_array)
#list_edges_partial = [ctpc_algo(n,total_vars_numb) for n in self.nodes]
return set(itertools.chain.from_iterable(list_edges_partial))
@timing
def estimate_structure(self,disable_multiprocessing:bool=False):
return self.ctpc_algorithm(disable_multiprocessing=disable_multiprocessing)

@ -1,189 +0,0 @@
import sys
sys.path.append('../')
import itertools
import json
import typing
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
from networkx.readwrite import json_graph
from abc import ABC
import abc
import ..utility.cache as ch
import ..structure_graph.conditional_intensity_matrix import ConditionalIntensityMatrix
import ..structure_graph.network_graph import NetworkGraph
import .parameters_estimator import ParametersEstimator
import ..structure_graph.sample_path import SamplePath
from ..structure_graph.structure import Structure
class StructureEstimator(object):
"""Has the task of estimating the network structure given the trajectories in ``samplepath``.
:param sample_path: the _sample_path object containing the trajectories and the real structure
:type sample_path: SamplePath
:_nodes: the nodes labels
:_nodes_vals: the nodes cardinalities
:_nodes_indxs: the nodes indexes
:_complete_graph: the complete directed graph built using the nodes labels in ``_nodes``
"""
def __init__(self, sample_path: SamplePath, known_edges: typing.List = None):
self._sample_path = sample_path
self.nodes = np.array(self._sample_path.structure.nodes_labels)
self.nodes_vals = self._sample_path.structure.nodes_values
self.nodes_indxs = self._sample_path.structure.nodes_indexes
self._removable_edges_matrix = self.
(known_edges)
self.complete_graph = self.build_complete_graph(self._sample_path.structure.nodes_labels)
self.cache = ch.Cache()
def build_removable_edges_matrix(self, known_edges: typing.List):
"""Builds a boolean matrix who shows if a edge could be removed or not, based on prior knowledge given:
:param known_edges: the list of nodes labels
:type known_edges: List
:return: a boolean matrix
:rtype: np.ndarray
"""
tot_vars_count = self._sample_path.total_variables_count
complete_adj_matrix = np.full((tot_vars_count, tot_vars_count), True)
if known_edges:
for edge in known_edges:
i = self._sample_path.structure.get_node_indx(edge[0])
j = self._sample_path.structure.get_node_indx(edge[1])
complete_adj_matrix[i][j] = False
return complete_adj_matrix
@staticmethod
def build_complete_graph(node_ids: typing.List) -> nx.DiGraph:
"""Builds a complete directed graph (no self loops) given the nodes labels in the list ``node_ids``:
:param node_ids: the list of nodes labels
:type node_ids: List
:return: a complete Digraph Object
:rtype: networkx.DiGraph
"""
complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids)
complete_graph.add_edges_from(itertools.permutations(node_ids, 2))
return complete_graph
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str):
"""Creates a list containing all possible subsets of the list ``u`` of size ``size``,
that do not contains a the node identified by ``parent_label``.
:param u: the list of nodes
:type u: List
:param size: the size of the subsets
:type size: int
:param parent_label: the node to exclude in the subsets generation
:type parent_label: string
:return: an Iterator Object containing a list of lists
:rtype: Iterator
"""
list_without_test_parent = u[:]
list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size))
def save_results(self) -> None:
"""Save the estimated Structure to a .json file in the path where the data are loaded from.
The file is named as the input dataset but the `results_` word is appended to the results file.
"""
res = json_graph.node_link_data(self._complete_graph)
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1]
name = name.split('.', 1)[0]
name += '_' + str(self._sample_path._importer.dataset_id())
name += '.json'
file_name = 'results_' + name
with open(file_name, 'w') as f:
json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)
@abc.abstractmethod
def estimate_structure(self) -> typing.List:
"""Abstract method to estimate the structure
:return: List of estimated edges
:rtype: Typing.List
"""
pass
def adjacency_matrix(self) -> np.ndarray:
"""Converts the estimated structure ``_complete_graph`` to a boolean adjacency matrix representation.
:return: The adjacency matrix of the graph ``_complete_graph``
:rtype: numpy.ndArray
"""
return nx.adj_matrix(self._complete_graph).toarray().astype(bool)
def spurious_edges(self) -> typing.List:
"""Return the spurious edges present in the estimated structure, if a prior net structure is present in
``_sample_path.structure``.
:return: A list containing the spurious edges
:rtype: List
"""
if not self._sample_path.has_prior_net_structure:
raise RuntimeError("Can not compute spurious edges with no prior net structure!")
real_graph = nx.DiGraph()
real_graph.add_nodes_from(self._sample_path.structure.nodes_labels)
real_graph.add_edges_from(self._sample_path.structure.edges)
return nx.difference(real_graph, self._complete_graph).edges
def save_plot_estimated_structure_graph(self) -> None:
"""Plot the estimated structure in a graphical model style.
Spurious edges are colored in red.
"""
graph_to_draw = nx.DiGraph()
spurious_edges = self.spurious_edges()
non_spurious_edges = list(set(self._complete_graph.edges) - set(spurious_edges))
print(non_spurious_edges)
edges_colors = ['red' if edge in spurious_edges else 'black' for edge in self._complete_graph.edges]
graph_to_draw.add_edges_from(spurious_edges)
graph_to_draw.add_edges_from(non_spurious_edges)
pos = nx.spring_layout(graph_to_draw, k=0.5*1/np.sqrt(len(graph_to_draw.nodes())), iterations=50,scale=10)
options = {
"node_size": 2000,
"node_color": "white",
"edgecolors": "black",
'linewidths':2,
"with_labels":True,
"font_size":13,
'connectionstyle': 'arc3, rad = 0.1',
"arrowsize": 15,
"arrowstyle": '<|-',
"width": 1,
"edge_color":edges_colors,
}
nx.draw(graph_to_draw, pos, **options)
ax = plt.gca()
ax.margins(0.20)
plt.axis("off")
name = self._sample_path._importer.file_path.rsplit('/', 1)[-1]
name = name.split('.', 1)[0]
name += '_' + str(self._sample_path._importer.dataset_id())
name += '.png'
plt.savefig(name)
plt.clf()
print("Estimated Structure Plot Saved At: ", os.path.abspath(name))

@ -23,7 +23,6 @@ from .fam_score_calculator import FamScoreCalculator
from ..optimizers.hill_climbing_search import HillClimbing
from ..optimizers.tabu_search import TabuSearch
from ..utility.decorators import timing,timing_write
import multiprocessing
from multiprocessing import Pool
@ -53,7 +52,6 @@ class StructureScoreBasedEstimator(StructureEstimator):
self.alpha_xu=alpha_xu
@timing
def estimate_structure(self, max_parents:int = None, iterations_number:int= 40,
patience:int = None, tabu_length:int = None, tabu_rules_duration:int = None,
optimizer: str = 'tabu',disable_multiprocessing:bool= False ):

@ -1,41 +0,0 @@
from abc import ABC, abstractmethod
import sys
sys.path.append('../')
import utility.abstract_importer as ai
class AbstractSamplePath(ABC):
def __init__(self, importer: ai.AbstractImporter):
self.importer = importer
self._trajectories = None
self._structure = None
super().__init__()
@abstractmethod
def build_trajectories(self):
"""
Builds the Trajectory object that will contain all the trajectories.
Assigns the Trajectoriy object to the instance attribute _trajectories
Clears all the unused dataframes in Importer Object
Parameters:
void
Returns:
void
"""
pass
@abstractmethod
def build_structure(self):
"""
Builds the Structure object that aggregates all the infos about the net.
Assigns the Structure object to the instance attribuite _structure
Parameters:
void
Returns:
void
"""
pass

@ -1,44 +0,0 @@
import numpy as np
import sys
sys.path.append('../')
class ConditionalIntensityMatrix:
"""Abstracts the Conditional Intesity matrix of a node as aggregation of the state residence times vector
and state transition matrix and the actual CIM matrix.
:param state_residence_times: state residence times vector
:type state_residence_times: numpy.array
:param state_transition_matrix: the transitions count matrix
:type state_transition_matrix: numpy.ndArray
:_cim: the actual cim of the node
"""
def __init__(self, state_residence_times: np.array, state_transition_matrix: np.array):
"""Constructor Method
"""
self._state_residence_times = state_residence_times
self._state_transition_matrix = state_transition_matrix
self._cim = self.state_transition_matrix.astype(np.float64)
def compute_cim_coefficients(self):
"""Compute the coefficients of the matrix _cim by using the following equality q_xx' = M[x, x'] / T[x].
The class member ``_cim`` will contain the computed cim
"""
np.fill_diagonal(self._cim, self._cim.diagonal() * -1)
self._cim = ((self._cim.T + 1) / (self._state_residence_times + 1)).T
@property
def state_residence_times(self):
return self._state_residence_times
@property
def state_transition_matrix(self):
return self._state_transition_matrix
@property
def cim(self):
return self._cim
def __repr__(self):
return 'CIM:\n' + str(self.cim)

@ -1,285 +0,0 @@
import typing
import networkx as nx
import numpy as np
from .structure import Structure
class NetworkGraph(object):
"""Abstracts the infos contained in the Structure class in the form of a directed graph.
Has the task of creating all the necessary filtering and indexing structures for parameters estimation
:param graph_struct: the ``Structure`` object from which infos about the net will be extracted
:type graph_struct: Structure
:_graph: directed graph
:_aggregated_info_about_nodes_parents: a structure that contains all the necessary infos
about every parents of the node of which all the indexing and filtering structures will be constructed.
:_time_scalar_indexing_structure: the indexing structure for state res time estimation
:_transition_scalar_indexing_structure: the indexing structure for transition computation
:_time_filtering: the columns filtering structure used in the computation of the state res times
:_transition_filtering: the columns filtering structure used in the computation of the transition
from one state to another
:_p_combs_structure: all the possible parents states combination for the node of interest
"""
def __init__(self, graph_struct: Structure):
"""Constructor Method
"""
self._graph_struct = graph_struct
self._graph = nx.DiGraph()
self._aggregated_info_about_nodes_parents = None
self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None
self._time_filtering = None
self._transition_filtering = None
self._p_combs_structure = None
def init_graph(self):
self.add_nodes(self._nodes_labels)
self.add_edges(self.graph_struct.edges)
self.aggregated_info_about_nodes_parents = self.get_ord_set_of_par_of_all_nodes()
self._fancy_indexing = self.build_fancy_indexing_structure(0)
self.build_scalar_indexing_structures()
self.build_time_columns_filtering_structure()
self.build_transition_columns_filtering_structure()
self._p_combs_structure = self.build_p_combs_structure()
def fast_init(self, node_id: str) -> None:
"""Initializes all the necessary structures for parameters estimation of the node identified by the label
node_id
:param node_id: the label of the node
:type node_id: string
"""
self.add_nodes(self._graph_struct.nodes_labels)
self.add_edges(self._graph_struct.edges)
self._aggregated_info_about_nodes_parents = self.get_ordered_by_indx_set_of_parents(node_id)
p_indxs = self._aggregated_info_about_nodes_parents[1]
p_vals = self._aggregated_info_about_nodes_parents[2]
node_states = self.get_states_number(node_id)
node_indx = self.get_node_indx(node_id)
cols_number = self._graph_struct.total_variables_number
self._time_scalar_indexing_structure = NetworkGraph.\
build_time_scalar_indexing_structure_for_a_node(node_states, p_vals)
self._transition_scalar_indexing_structure = NetworkGraph.\
build_transition_scalar_indexing_structure_for_a_node(node_states, p_vals)
self._time_filtering = NetworkGraph.build_time_columns_filtering_for_a_node(node_indx, p_indxs)
self._transition_filtering = NetworkGraph.build_transition_filtering_for_a_node(node_indx, p_indxs, cols_number)
self._p_combs_structure = NetworkGraph.build_p_comb_structure_for_a_node(p_vals)
def add_nodes(self, list_of_nodes: typing.List) -> None:
"""Adds the nodes to the ``_graph`` contained in the list of nodes ``list_of_nodes``.
Sets all the properties that identify a nodes (index, positional index, cardinality)
:param list_of_nodes: the nodes to add to ``_graph``
:type list_of_nodes: List
"""
nodes_indxs = self._graph_struct.nodes_indexes
nodes_vals = self._graph_struct.nodes_values
pos = 0
for id, node_indx, node_val in zip(list_of_nodes, nodes_indxs, nodes_vals):
self._graph.add_node(id, indx=node_indx, val=node_val, pos_indx=pos)
pos += 1
def has_edge(self,edge:tuple)-> bool:
"""
Check if the graph contains a specific edge
Parameters:
edge: a tuple that rappresents the edge
Returns:
bool
"""
return self.graph.has_edge(edge[0],edge[1])
def add_edges(self, list_of_edges: typing.List) -> None:
"""Add the edges to the ``_graph`` contained in the list ``list_of_edges``.
:param list_of_edges: the list containing of tuples containing the edges
:type list_of_edges: List
"""
self._graph.add_edges_from(list_of_edges)
def remove_node(self, node_id: str) -> None:
"""Remove the node ``node_id`` from all the class members.
Initialize all the filtering/indexing structures.
"""
self._graph.remove_node(node_id)
self._graph_struct.remove_node(node_id)
self.clear_indexing_filtering_structures()
def clear_indexing_filtering_structures(self) -> None:
"""Initialize all the filtering/indexing structures.
"""
self._aggregated_info_about_nodes_parents = None
self._time_scalar_indexing_structure = None
self._transition_scalar_indexing_structure = None
self._time_filtering = None
self._transition_filtering = None
self._p_combs_structure = None
def get_ordered_by_indx_set_of_parents(self, node: str) -> typing.Tuple:
"""Builds the aggregated structure that holds all the infos relative to the parent set of the node, namely
(parents_labels, parents_indexes, parents_cardinalities).
:param node: the label of the node
:type node: string
:return: a tuple containing all the parent set infos
:rtype: Tuple
"""
parents = self.get_parents_by_id(node)
nodes = self._graph_struct.nodes_labels
d = {v: i for i, v in enumerate(nodes)}
sorted_parents = sorted(parents, key=lambda v: d[v])
get_node_indx = self.get_node_indx
p_indxes = [get_node_indx(node) for node in sorted_parents]
p_values = [self.get_states_number(node) for node in sorted_parents]
return sorted_parents, p_indxes, p_values
@staticmethod
def build_time_scalar_indexing_structure_for_a_node(node_states: int,
parents_vals: typing.List) -> np.ndarray:
"""Builds an indexing structure for the computation of state residence times values.
:param node_states: the node cardinality
:type node_states: int
:param parents_vals: the caridinalites of the node's parents
:type parents_vals: List
:return: The time indexing structure
:rtype: numpy.ndArray
"""
T_vector = np.array([node_states])
T_vector = np.append(T_vector, parents_vals)
T_vector = T_vector.cumprod().astype(np.int)
return T_vector
@staticmethod
def build_transition_scalar_indexing_structure_for_a_node(node_states_number: int, parents_vals: typing.List) \
-> np.ndarray:
"""Builds an indexing structure for the computation of state transitions values.
:param node_states_number: the node cardinality
:type node_states_number: int
:param parents_vals: the caridinalites of the node's parents
:type parents_vals: List
:return: The transition indexing structure
:rtype: numpy.ndArray
"""
M_vector = np.array([node_states_number,
node_states_number])
M_vector = np.append(M_vector, parents_vals)
M_vector = M_vector.cumprod().astype(np.int)
return M_vector
@staticmethod
def build_time_columns_filtering_for_a_node(node_indx: int, p_indxs: typing.List) -> np.ndarray:
"""
Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
in the dataset.
This structute will be used in the computation of the state res times.
:param node_indx: the index of the node
:type node_indx: int
:param p_indxs: the indexes of the node's parents
:type p_indxs: List
:return: The filtering structure for times estimation
:rtype: numpy.ndArray
"""
return np.append(np.array([node_indx], dtype=np.int), p_indxs).astype(np.int)
@staticmethod
def build_transition_filtering_for_a_node(node_indx: int, p_indxs: typing.List, nodes_number: int) \
-> np.ndarray:
"""Builds the necessary structure to filter the desired columns indicated by ``node_indx`` and ``p_indxs``
in the dataset.
This structure will be used in the computation of the state transitions values.
:param node_indx: the index of the node
:type node_indx: int
:param p_indxs: the indexes of the node's parents
:type p_indxs: List
:param nodes_number: the total number of nodes in the dataset
:type nodes_number: int
:return: The filtering structure for transitions estimation
:rtype: numpy.ndArray
"""
return np.array([node_indx + nodes_number, node_indx, *p_indxs], dtype=np.int)
@staticmethod
def build_p_comb_structure_for_a_node(parents_values: typing.List) -> np.ndarray:
"""
Builds the combinatorial structure that contains the combinations of all the values contained in
``parents_values``.
:param parents_values: the cardinalities of the nodes
:type parents_values: List
:return: A numpy matrix containing a grid of the combinations
:rtype: numpy.ndArray
"""
tmp = []
for val in parents_values:
tmp.append([x for x in range(val)])
if len(parents_values) > 0:
parents_comb = np.array(np.meshgrid(*tmp)).T.reshape(-1, len(parents_values))
if len(parents_values) > 1:
tmp_comb = parents_comb[:, 1].copy()
parents_comb[:, 1] = parents_comb[:, 0].copy()
parents_comb[:, 0] = tmp_comb
else:
parents_comb = np.array([[]], dtype=np.int)
return parents_comb
def get_parents_by_id(self, node_id) -> typing.List:
"""Returns a list of labels of the parents of the node ``node_id``
:param node_id: the node label
:type node_id: string
:return: a List of labels of the parents
:rtype: List
"""
return list(self._graph.predecessors(node_id))
def get_states_number(self, node_id) -> int:
return self._graph.nodes[node_id]['val']
def get_node_indx(self, node_id) -> int:
return nx.get_node_attributes(self._graph, 'indx')[node_id]
def get_positional_node_indx(self, node_id) -> int:
return self._graph.nodes[node_id]['pos_indx']
@property
def nodes(self) -> typing.List:
return self._graph_struct.nodes_labels
@property
def edges(self) -> typing.List:
return list(self._graph.edges)
@property
def nodes_indexes(self) -> np.ndarray:
return self._graph_struct.nodes_indexes
@property
def nodes_values(self) -> np.ndarray:
return self._graph_struct.nodes_values
@property
def time_scalar_indexing_strucure(self) -> np.ndarray:
return self._time_scalar_indexing_structure
@property
def time_filtering(self) -> np.ndarray:
return self._time_filtering
@property
def transition_scalar_indexing_structure(self) -> np.ndarray:
return self._transition_scalar_indexing_structure
@property
def transition_filtering(self) -> np.ndarray:
return self._transition_filtering
@property
def p_combs(self) -> np.ndarray:
return self._p_combs_structure

@ -1,95 +0,0 @@
import sys
sys.path.append('../')
import numpy as np
import pandas as pd
import .abstract_sample_path as asam
import ..utility.json_importer as imp
from .structure import Structure
from .trajectory import Trajectory
import ..utility.abstract_importer as ai
class SamplePath(object):
"""Aggregates all the informations about the trajectories, the real structure of the sampled net and variables
cardinalites. Has the task of creating the objects ``Trajectory`` and ``Structure`` that will
contain the mentioned data.
:param importer: the Importer object which contains the imported and processed data
:type importer: AbstractImporter
:_trajectories: the ``Trajectory`` object that will contain all the concatenated trajectories
:_structure: the ``Structure`` Object that will contain all the structural infos about the net
:_total_variables_count: the number of variables in the net
"""
def __init__(self, importer: ai.AbstractImporter):
"""Constructor Method
"""
self._importer = importer
if self._importer._df_variables is None or self._importer._concatenated_samples is None:
raise RuntimeError('The importer object has to contain the all processed data!')
if self._importer._df_variables.empty:
raise RuntimeError('The importer object has to contain the all processed data!')
if isinstance(self._importer._concatenated_samples, pd.DataFrame):
if self._importer._concatenated_samples.empty:
raise RuntimeError('The importer object has to contain the all processed data!')
if isinstance(self._importer._concatenated_samples, np.ndarray):
if self._importer._concatenated_samples.size == 0:
raise RuntimeError('The importer object has to contain the all processed data!')
self._trajectories = None
self._structure = None
self._total_variables_count = None
def build_trajectories(self) -> None:
"""Builds the Trajectory object that will contain all the trajectories.
Clears all the unused dataframes in ``_importer`` Object
"""
self._trajectories = \
Trajectory(self._importer.build_list_of_samples_array(self._importer.concatenated_samples),
len(self._importer.sorter) + 1)
self._importer.clear_concatenated_frame()
def build_structure(self) -> None:
"""
Builds the ``Structure`` object that aggregates all the infos about the net.
"""
if self._importer.sorter != self._importer.variables.iloc[:, 0].to_list():
raise RuntimeError("The Dataset columns order have to match the order of labels in the variables Frame!")
self._total_variables_count = len(self._importer.sorter)
labels = self._importer.variables.iloc[:, 0].to_list()
indxs = self._importer.variables.index.to_numpy()
vals = self._importer.variables.iloc[:, 1].to_numpy()
if self._importer.structure is None or self._importer.structure.empty:
edges = []
else:
edges = list(self._importer.structure.to_records(index=False))
self._structure = Structure(labels, indxs, vals, edges,
self._total_variables_count)
def clear_memory(self):
self._importer._raw_data = []
@property
def trajectories(self) -> Trajectory:
return self._trajectories
@property
def structure(self) -> Structure:
return self._structure
@property
def total_variables_count(self) -> int:
return self._total_variables_count
@property
def has_prior_net_structure(self) -> bool:
return bool(self._structure.edges)

@ -1,98 +0,0 @@
import sys
sys.path.append('../')
import typing
import numpy as np
import structure_graph.conditional_intensity_matrix as cim
class SetOfCims(object):
"""Aggregates all the CIMS of the node identified by the label _node_id.
:param node_id: the node label
:type node_ind: string
:param parents_states_number: the cardinalities of the parents
:type parents_states_number: List
:param node_states_number: the caridinality of the node
:type node_states_number: int
:param p_combs: the p_comb structure bound to this node
:type p_combs: numpy.ndArray
:_state_residence_time: matrix containing all the state residence time vectors for the node
:_transition_matrices: matrix containing all the transition matrices for the node
:_actual_cims: the cims of the node
"""
def __init__(self, node_id: str, parents_states_number: typing.List, node_states_number: int, p_combs: np.ndarray):
"""Constructor Method
"""
self._node_id = node_id
self._parents_states_number = parents_states_number
self._node_states_number = node_states_number
self._actual_cims = []
self._state_residence_times = None
self._transition_matrices = None
self._p_combs = p_combs
self.build_times_and_transitions_structures()
def build_times_and_transitions_structures(self) -> None:
"""Initializes at the correct dimensions the state residence times matrix and the state transition matrices.
"""
if not self._parents_states_number:
self._state_residence_times = np.zeros((1, self._node_states_number), dtype=np.float)
self._transition_matrices = np.zeros((1, self._node_states_number, self._node_states_number), dtype=np.int)
else:
self._state_residence_times = \
np.zeros((np.prod(self._parents_states_number), self._node_states_number), dtype=np.float)
self._transition_matrices = np.zeros([np.prod(self._parents_states_number), self._node_states_number,
self._node_states_number], dtype=np.int)
def build_cims(self, state_res_times: np.ndarray, transition_matrices: np.ndarray) -> None:
"""Build the ``ConditionalIntensityMatrix`` objects given the state residence times and transitions matrices.
Compute the cim coefficients.The class member ``_actual_cims`` will contain the computed cims.
:param state_res_times: the state residence times matrix
:type state_res_times: numpy.ndArray
:param transition_matrices: the transition matrices
:type transition_matrices: numpy.ndArray
"""
for state_res_time_vector, transition_matrix in zip(state_res_times, transition_matrices):
cim_to_add = cim.ConditionalIntensityMatrix(state_res_time_vector, transition_matrix)
cim_to_add.compute_cim_coefficients()
self._actual_cims.append(cim_to_add)
self._actual_cims = np.array(self._actual_cims)
self._transition_matrices = None
self._state_residence_times = None
def filter_cims_with_mask(self, mask_arr: np.ndarray, comb: typing.List) -> np.ndarray:
"""Filter the cims contained in the array ``_actual_cims`` given the boolean mask ``mask_arr`` and the index
``comb``.
:param mask_arr: the boolean mask that indicates which parent to consider
:type mask_arr: numpy.array
:param comb: the state/s of the filtered parents
:type comb: numpy.array
:return: Array of ``ConditionalIntensityMatrix`` objects
:rtype: numpy.array
"""
if mask_arr.size <= 1:
return self._actual_cims
else:
flat_indxs = np.argwhere(np.all(self._p_combs[:, mask_arr] == comb, axis=1)).ravel()
return self._actual_cims[flat_indxs]
@property
def actual_cims(self) -> np.ndarray:
return self._actual_cims
@property
def p_combs(self) -> np.ndarray:
return self._p_combs
def get_cims_number(self):
return len(self._actual_cims)

@ -1,28 +0,0 @@
import sys
sys.path.append('../')
import structure_graph.set_of_cims as socim
class SetsOfCimsContainer:
"""
Aggrega un insieme di oggetti SetOfCims
"""
def __init__(self, list_of_keys, states_number_per_node, list_of_parents_states_number, p_combs_list):
self.sets_of_cims = None
self.init_cims_structure(list_of_keys, states_number_per_node, list_of_parents_states_number, p_combs_list)
#self.states_per_variable = states_number
def init_cims_structure(self, keys, states_number_per_node, list_of_parents_states_number, p_combs_list):
"""for indx, key in enumerate(keys):
self.sets_of_cims.append(
socim.SetOfCims(key, list_of_parents_states_number[indx], states_number_per_node[indx]))"""
self.sets_of_cims = [socim.SetOfCims(pair[1], list_of_parents_states_number[pair[0]], states_number_per_node[pair[0]], p_combs_list[pair[0]])
for pair in enumerate(keys)]
def get_set_of_cims(self, node_indx):
return self.sets_of_cims[node_indx]
def get_cims_of_node(self, node_indx, cim_indx):
return self.sets_of_cims[node_indx].get_cim(cim_indx)

@ -1,128 +0,0 @@
import sys
sys.path.append('../')
import typing as ty
import numpy as np
class Structure(object):
"""Contains all the infos about the network structure(nodes labels, nodes caridinalites, edges, indexes)
:param nodes_labels_list: the symbolic names of the variables
:type nodes_labels_list: List
:param nodes_indexes_arr: the indexes of the nodes
:type nodes_indexes_arr: numpy.ndArray
:param nodes_vals_arr: the cardinalites of the nodes
:type nodes_vals_arr: numpy.ndArray
:param edges_list: the edges of the network
:type edges_list: List
:param total_variables_number: the total number of variables in the dataset
:type total_variables_number: int
"""
def __init__(self, nodes_labels_list: ty.List, nodes_indexes_arr: np.ndarray, nodes_vals_arr: np.ndarray,
edges_list: ty.List, total_variables_number: int):
"""Constructor Method
"""
self._nodes_labels_list = nodes_labels_list
self._nodes_indexes_arr = nodes_indexes_arr
self._nodes_vals_arr = nodes_vals_arr
self._edges_list = edges_list
self._total_variables_number = total_variables_number
def remove_node(self, node_id: str) -> None:
"""Remove the node ``node_id`` from all the class members.
The class member ``_total_variables_number`` since it refers to the total number of variables in the dataset.
"""
node_positional_indx = self._nodes_labels_list.index(node_id)
del self._nodes_labels_list[node_positional_indx]
self._nodes_indexes_arr = np.delete(self._nodes_indexes_arr, node_positional_indx)
self._nodes_vals_arr = np.delete(self._nodes_vals_arr, node_positional_indx)
self._edges_list = [(from_node, to_node) for (from_node, to_node) in self._edges_list if (from_node != node_id
and to_node != node_id)]
@property
def edges(self) -> ty.List:
return self._edges_list
@property
def nodes_labels(self) -> ty.List:
return self._nodes_labels_list
@property
def nodes_indexes(self) -> np.ndarray:
return self._nodes_indexes_arr
@property
def nodes_values(self) -> np.ndarray:
return self._nodes_vals_arr
@property
def total_variables_number(self) -> int:
return self._total_variables_number
def get_node_id(self, node_indx: int) -> str:
"""
Given the ``node_index`` returns the node label.
:param node_indx: the node index
:type node_indx: int
:return: the node label
:rtype: string
"""
return self._nodes_labels_list[node_indx]
def clean_structure_edges(self):
self._edges_list = list()
def add_edge(self,edge: tuple):
self._edges_list.append(tuple)
print(self._edges_list)
def remove_edge(self,edge: tuple):
self._edges_list.remove(tuple)
def contains_edge(self,edge:tuple) -> bool:
return edge in self._edges_list
def get_node_indx(self, node_id: str) -> int:
"""
Given the ``node_index`` returns the node label.
:param node_id: the node label
:type node_id: string
:return: the node index
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node_id)
return self._nodes_indexes_arr[pos_indx]
def get_positional_node_indx(self, node_id: str) -> int:
return self._nodes_labels_list.index(node_id)
def get_states_number(self, node: str) -> int:
"""Given the node label ``node`` returns the cardinality of the node.
:param node: the node label
:type node: string
:return: the node cardinality
:rtype: int
"""
pos_indx = self._nodes_labels_list.index(node)
return self._nodes_vals_arr[pos_indx]
def __repr__(self):
return "Variables:\n" + str(self._nodes_labels_list) +"\nValues:\n"+ str(self._nodes_vals_arr) +\
"\nEdges: \n" + str(self._edges_list)
def __eq__(self, other):
"""Overrides the default implementation"""
if isinstance(other, Structure):
return set(self._nodes_labels_list) == set(other._nodes_labels_list) and \
np.array_equal(self._nodes_vals_arr, other._nodes_vals_arr) and \
np.array_equal(self._nodes_indexes_arr, other._nodes_indexes_arr) and \
self._edges_list == other._edges_list
return False

@ -1,28 +0,0 @@
from functools import wraps
from time import time
def timing(f):
@wraps(f)
def wrap(*args, **kw):
ts = time()
result = f(*args, **kw)
te = time()
print (f"{f.__name__} args:[{args},{kw}] took: {te-ts} sec")
return result
return wrap
def timing_write(f):
@wraps(f)
def wrap(*args, **kw):
ts = time()
result = f(*args, **kw)
te = time()
print (f"{f.__name__} args:[{args},{kw}] took: {te-ts} sec")
with open("../results/results.csv", 'a+') as fi:
fi.write(f"\n{round(te-ts,3)},")
return result
return wrap

File diff suppressed because it is too large Load Diff

@ -1,14 +0,0 @@
Time,Type,Variables,Density_Network,Cardinality,Index,F1,Precision,Recall
4.19,
4.078,
4.368,
6.024,
8.198,
10.586,
10.589,
10.447,
10.516,
8.335,
11.243,
11.78,
1 Time,Type,Variables,Density_Network,Cardinality,Index,F1,Precision,Recall
2 4.19,
3 4.078,
4 4.368,
5 6.024,
6 8.198,
7 10.586,
8 10.589,
9 10.447,
10 10.516,
11 8.335,
12 11.243,
13 11.78,

@ -1,20 +0,0 @@
6.255,0,3,0.2,2,0,1.0,1.0,1.0
7.305,0,4,0.2,2,0,1.0,1.0,1.0
13.017,0,5,0.2,2,0,1.0,1.0,1.0
18.742,0,6,0.2,2,0,1.0,1.0,1.0
47.785,0,10,0.2,2,0,1.0,1.0,1.0
161.731,0,15,0.2,2,0,1.0,1.0,1.0
284.029,0,20,0.2,2,0,0.989,0.978,1.0
7.5,0,3,0.2,3,0,1.0,1.0,1.0
11.541,0,4,0.2,3,0,1.0,1.0,1.0
19.318,0,5,0.2,3,0,1.0,1.0,1.0
29.8,0,6,0.2,3,0,1.0,1.0,1.0
106.663,0,10,0.2,3,0,1.0,1.0,1.0
265.436,0,15,0.2,3,0,0.876,1.0,0.78
511.009,0,20,0.2,3,0,0.95,1.0,0.905
9.866,0,3,0.2,4,0,1.0,1.0,1.0
16.127,0,4,0.2,4,0,1.0,1.0,1.0
25.694,0,5,0.2,4,0,1.0,1.0,1.0
44.268,0,6,0.2,4,0,1.0,1.0,1.0
146.875,0,10,0.2,4,0,1.0,1.0,1.0
403.063,0,15,0.2,4,0,0.935,1.0,0.878
1 6.255 0 3 0.2 2 0 1.0 1.0 1.0
2 7.305 0 4 0.2 2 0 1.0 1.0 1.0
3 13.017 0 5 0.2 2 0 1.0 1.0 1.0
4 18.742 0 6 0.2 2 0 1.0 1.0 1.0
5 47.785 0 10 0.2 2 0 1.0 1.0 1.0
6 161.731 0 15 0.2 2 0 1.0 1.0 1.0
7 284.029 0 20 0.2 2 0 0.989 0.978 1.0
8 7.5 0 3 0.2 3 0 1.0 1.0 1.0
9 11.541 0 4 0.2 3 0 1.0 1.0 1.0
10 19.318 0 5 0.2 3 0 1.0 1.0 1.0
11 29.8 0 6 0.2 3 0 1.0 1.0 1.0
12 106.663 0 10 0.2 3 0 1.0 1.0 1.0
13 265.436 0 15 0.2 3 0 0.876 1.0 0.78
14 511.009 0 20 0.2 3 0 0.95 1.0 0.905
15 9.866 0 3 0.2 4 0 1.0 1.0 1.0
16 16.127 0 4 0.2 4 0 1.0 1.0 1.0
17 25.694 0 5 0.2 4 0 1.0 1.0 1.0
18 44.268 0 6 0.2 4 0 1.0 1.0 1.0
19 146.875 0 10 0.2 4 0 1.0 1.0 1.0
20 403.063 0 15 0.2 4 0 0.935 1.0 0.878

File diff suppressed because it is too large Load Diff

@ -1,147 +0,0 @@
import sys
sys.path.append("../../classes/")
import glob
import math
import os
import unittest
import networkx as nx
import numpy as np
import pandas as pd
import psutil
from line_profiler import LineProfiler
import copy
import json
import utility.cache as ch
import structure_graph.sample_path as sp
import estimators.structure_score_based_estimator as se_score
import estimators.structure_constraint_based_estimator as se_constr
import utility.sample_importer as si
class TestTabuSearch(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
def test_constr(self):
list_constraint= [0,1]
list_cardinality= [[2,"binary"],[3,"ternary"], [4,"quaternary"]]
list_dens = [["0.1","_01"],["0.2","_02"], ["0.3",""], ["0.4","_04"] ]
for constr in list_constraint:
for card in list_cardinality:
for dens in list_dens:
if card[0] == 4:
list_vals= [3,4,5,6,10,15]
else:
list_vals= [3,4,5,6,10,15,20]
for var_n in list_vals:
patience = 25
var_number= var_n
if var_number > 11:
patience = 30
if var_number > 16:
patience = 35
cardinality = card[0]
cardinality_string = card[1]
density= dens[0]
density_string = dens[1]
constraint = constr
index = 1
num_networks=10
while index <= num_networks:
with open(f"/home/alessandro/Documents/ctbn_cba/data/networks_and_trajectories_{cardinality_string}_data{density_string}_{var_number}/{index}.json") as f:
raw_data = json.load(f)
trajectory_list_raw= raw_data["samples"]
trajectory_list = [pd.DataFrame(sample) for sample in trajectory_list_raw]
variables= pd.DataFrame(raw_data["variables"])
prior_net_structure = pd.DataFrame(raw_data["dyn.str"])
self.importer = si.SampleImporter(
trajectory_list=trajectory_list,
variables=variables,
prior_net_structure=prior_net_structure
)
self.importer.import_data()
self.s1 = sp.SamplePath(self.importer)
self.s1.build_trajectories()
self.s1.build_structure()
true_edges = copy.deepcopy(self.s1.structure.edges)
true_edges = set(map(tuple, true_edges))
if constr == 0:
se1 = se_score.StructureScoreBasedEstimator(self.s1)
set_list_edges = se1.estimate_structure(
max_parents = None,
iterations_number = 100,
patience = patience,
tabu_length = var_number,
tabu_rules_duration = var_number,
optimizer = 'tabu'
)
else:
se1 = se_constr.StructureConstraintBasedEstimator(self.s1,0.1,0.1)
set_list_edges = se1.estimate_structure(disable_multiprocessing=False)
n_added_fake_edges = len(set_list_edges.difference(true_edges))
n_missing_edges = len(true_edges.difference(set_list_edges))
n_true_positive = len(true_edges) - n_missing_edges
precision = n_true_positive / (n_true_positive + n_added_fake_edges)
recall = n_true_positive / (n_true_positive + n_missing_edges)
f1_measure = round(2* (precision*recall) / (precision+recall),3)
print(true_edges)
print(set_list_edges)
print(f"precision: {precision} ")
print(f"recall: {recall} ")
with open("../results/results.csv", 'a+') as fi:
fi.write(f"{constraint},{var_number},{density},{cardinality},{index},{f1_measure},{round(precision,3)},{round(recall,3)}")
index += 1
self.assertEqual(set_list_edges, true_edges)
if __name__ == '__main__':
unittest.main()

@ -1,108 +0,0 @@
import sys
sys.path.append("../../classes/")
import glob
import math
import os
import unittest
import networkx as nx
import numpy as np
import psutil
from line_profiler import LineProfiler
import copy
import utility.cache as ch
import structure_graph.sample_path as sp
import estimators.structure_constraint_based_estimator as se
import utility.json_importer as ji
class TestTabuSearch(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
def test_constr(self):
list_vals= [3,4,5,6,10,15]
list_dens = [["0.1","_01"],["0.2","_02"], ["0.3",""], ["0.4","_04"] ]
for dens in list_dens:
for var_n in list_vals:
var_number= var_n
cardinality = 4
cardinality_string = "quaternary"
density= dens[0]
density_string = dens[1]
constraint = 1
index = 0
num_networks=10
if var_number > 9:
num_networks=3
while index < num_networks:
#cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
self.importer = ji.JsonImporter(f"../../data/networks_and_trajectories_{cardinality_string}_data{density_string}_{var_number}.json",
'samples', 'dyn.str', 'variables', 'Time', 'Name', index )
self.s1 = sp.SamplePath(self.importer)
self.s1.build_trajectories()
self.s1.build_structure()
true_edges = copy.deepcopy(self.s1.structure.edges)
true_edges = set(map(tuple, true_edges))
se1 = se.StructureConstraintBasedEstimator(self.s1, 0.1, 0.1)
se1.ctpc_algorithm()
set_list_edges = set(se1.complete_graph.edges)
n_added_fake_edges = len(set_list_edges.difference(true_edges))
n_missing_edges = len(true_edges.difference(set_list_edges))
n_true_positive = len(true_edges) - n_missing_edges
precision = n_true_positive / (n_true_positive + n_added_fake_edges)
recall = n_true_positive / (n_true_positive + n_missing_edges)
f1_measure = round(2* (precision*recall) / (precision+recall),3)
# print(f"n archi reali non trovati: {n_missing_edges}")
# print(f"n archi non reali aggiunti: {n_added_fake_edges}")
print(true_edges)
print(set_list_edges)
print(f"precision: {precision} ")
print(f"recall: {recall} ")
with open("../results/results.csv", 'a+') as fi:
fi.write(f"{constraint},{var_number},{density},{cardinality},{index},{f1_measure},{round(precision,3)},{round(recall,3)}")
index += 1
self.assertEqual(set_list_edges, true_edges)
if __name__ == '__main__':
unittest.main()

@ -1,122 +0,0 @@
import sys
sys.path.append("../../classes/")
import glob
import math
import os
import unittest
import networkx as nx
import numpy as np
import psutil
from line_profiler import LineProfiler
import copy
import utility.cache as ch
import structure_graph.sample_path as sp
import estimators.structure_score_based_estimator as se
import utility.json_importer as ji
class TestTabuSearch(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
def test_constr(self):
list_vals= [3,4,5,6,10,15]
list_dens = [["0.1","_01"],["0.2","_02"], ["0.3",""], ["0.4","_04"] ]
for dens in list_dens:
for var_n in list_vals:
patience = 20
var_number= var_n
if var_number > 11:
patience = 25
if var_number > 16:
patience = 35
cardinality = 4
cardinality_string = "quaternary"
density= dens[0]
density_string = dens[1]
constraint = 0
index = 0
num_networks=10
if var_number > 9:
num_networks=3
while index < num_networks:
#cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
self.importer = ji.JsonImporter(f"../../data/networks_and_trajectories_{cardinality_string}_data{density_string}_{var_number}.json",
'samples', 'dyn.str', 'variables', 'Time', 'Name', index )
self.s1 = sp.SamplePath(self.importer)
self.s1.build_trajectories()
self.s1.build_structure()
true_edges = copy.deepcopy(self.s1.structure.edges)
true_edges = set(map(tuple, true_edges))
se1 = se.StructureScoreBasedEstimator(self.s1)
set_list_edges = se1.estimate_structure(
max_parents = None,
iterations_number = 100,
patience = patience,
tabu_length = var_number,
tabu_rules_duration = var_number,
optimizer = 'hill'
)
n_added_fake_edges = len(set_list_edges.difference(true_edges))
n_missing_edges = len(true_edges.difference(set_list_edges))
n_true_positive = len(true_edges) - n_missing_edges
precision = n_true_positive / (n_true_positive + n_added_fake_edges)
recall = n_true_positive / (n_true_positive + n_missing_edges)
f1_measure = round(2* (precision*recall) / (precision+recall),3)
# print(f"n archi reali non trovati: {n_missing_edges}")
# print(f"n archi non reali aggiunti: {n_added_fake_edges}")
print(true_edges)
print(set_list_edges)
print(f"precision: {precision} ")
print(f"recall: {recall} ")
with open("../results/results.csv", 'a+') as fi:
fi.write(f"{constraint},{var_number},{density},{cardinality},{index},{f1_measure},{round(precision,3)},{round(recall,3)}")
index += 1
self.assertEqual(set_list_edges, true_edges)
if __name__ == '__main__':
unittest.main()