1
0
Fork 0

Refactoring classes Structure_estimator and organized files in folders

master
Luca Moretti 4 years ago
parent 5a57501164
commit 3165928ebd
  1. 10
      main_package/basic_main.py
  2. 9
      main_package/classes/estimators/fam_score_calculator.py
  3. 11
      main_package/classes/estimators/parameters_estimator.py
  4. 84
      main_package/classes/estimators/structure_constraint_based_estimator.py
  5. 96
      main_package/classes/estimators/structure_estimator.py
  6. 74
      main_package/classes/estimators/structure_score_based_estimator.py
  7. 5
      main_package/classes/structure_graph/abstract_sample_path.py
  8. 2
      main_package/classes/structure_graph/conditional_intensity_matrix.py
  9. 2
      main_package/classes/structure_graph/network_graph.py
  10. 12
      main_package/classes/structure_graph/sample_path.py
  11. 5
      main_package/classes/structure_graph/set_of_cims.py
  12. 5
      main_package/classes/structure_graph/sets_of_cims_container.py
  13. 3
      main_package/classes/structure_graph/structure.py
  14. 2
      main_package/classes/structure_graph/trajectory.py
  15. 0
      main_package/classes/utility/abstract_importer.py
  16. 5
      main_package/classes/utility/cache.py
  17. 0
      main_package/classes/utility/decorators.py
  18. 4
      main_package/classes/utility/json_importer.py
  19. 14
      main_package/tests/estimators/test_parameters_estimator.py
  20. 4
      main_package/tests/estimators/test_structure.py
  21. 24
      main_package/tests/estimators/test_structure_constraint_based_estimator.py
  22. 12
      main_package/tests/estimators/test_structure_score_based_estimator.py
  23. 4
      main_package/tests/structure_graph/test_cim.py
  24. 10
      main_package/tests/structure_graph/test_networkgraph.py
  25. 12
      main_package/tests/structure_graph/test_sample_path.py
  26. 4
      main_package/tests/structure_graph/test_setofcims.py
  27. 6
      main_package/tests/structure_graph/test_sets_of_cims_container.py
  28. 4
      main_package/tests/structure_graph/test_trajectory.py
  29. 6
      main_package/tests/utility/test_cache.py
  30. 6
      main_package/tests/utility/test_json_importer.py

@ -4,11 +4,11 @@ import os
import sys
sys.path.append("./classes/")
import network_graph as ng
import sample_path as sp
import set_of_cims as sofc
import parameters_estimator as pe
import json_importer as ji
import structure_graph.network_graph as ng
import structure_graph.sample_path as sp
import structure_graph.set_of_cims as sofc
import estimators.parameters_estimator as pe
import utility.json_importer as ji
def main():

@ -1,3 +1,6 @@
import sys
sys.path.append('../')
import itertools
import json
@ -12,9 +15,9 @@ from math import log
from scipy.special import loggamma
from random import choice
import set_of_cims as soCims
import network_graph as net_graph
import conditional_intensity_matrix as cim_class
import structure_graph.set_of_cims as soCims
import structure_graph.network_graph as net_graph
import structure_graph.conditional_intensity_matrix as cim_class
'''

@ -1,10 +1,11 @@
import sys
sys.path.append('../')
import numpy as np
import network_graph as ng
import sample_path as sp
import set_of_cims as sofc
import sets_of_cims_container as acims
import structure_graph.network_graph as ng
import structure_graph.sample_path as sp
import structure_graph.set_of_cims as sofc
import structure_graph.sets_of_cims_container as acims
class ParametersEstimator:

@ -1,4 +1,5 @@
import sys
sys.path.append('../')
import itertools
import json
import typing
@ -9,52 +10,28 @@ from networkx.readwrite import json_graph
from scipy.stats import chi2 as chi2_dist
from scipy.stats import f as f_dist
import cache as ch
import conditional_intensity_matrix as condim
import network_graph as ng
import parameters_estimator as pe
import sample_path as sp
import structure as st
import utility.cache as ch
import structure_graph.conditional_intensity_matrix as condim
import structure_graph.network_graph as ng
import estimators.parameters_estimator as pe
import estimators.structure_estimator as se
import structure_graph.sample_path as sp
import structure_graph.structure as st
class StructureEstimator:
class StructureConstraintBasedEstimator(se.StructureEstimator):
"""
Has the task of estimating the network structure given the trajectories in samplepath.
:sample_path: the sample_path object containing the trajectories and the real structure
:exp_test_sign: the significance level for the exponential Hp test
:chi_test_alfa: the significance level for the chi Hp test
:nodes: the nodes labels
:nodes_vals: the nodes cardinalities
:nodes_indxs: the nodes indexes
:complete_graph: the complete directed graph built using the nodes labels in nodes
:cache: the cache object
"""
def __init__(self, sample_path: sp.SamplePath, exp_test_alfa: float, chi_test_alfa: float):
self.sample_path = sample_path
self.nodes = np.array(self.sample_path.structure.nodes_labels)
self.nodes_vals = self.sample_path.structure.nodes_values
self.nodes_indxs = self.sample_path.structure.nodes_indexes
self.complete_graph = self.build_complete_graph(self.sample_path.structure.nodes_labels)
super().__init__(sample_path)
self.exp_test_sign = exp_test_alfa
self.chi_test_alfa = chi_test_alfa
self.cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List):
"""
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
Parameters:
node_ids: the list of nodes labels
Returns:
a complete Digraph Object
"""
complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids)
complete_graph.add_edges_from(itertools.permutations(node_ids, 2))
return complete_graph
def complete_test(self, test_parent: str, test_child: str, parent_set: typing.List, child_states_numb: int,
tot_vars_count: int):
@ -260,23 +237,6 @@ class StructureEstimator:
b += 1
self.cache.clear()
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str):
"""
Creates a list containing all possible subsets of the list u of size size,
that do not contains a the node identified by parent_label.
Parameters:
u: the list of nodes
size: the size of the subsets
parent_label: the nodes to exclude in the subsets generation
Returns:
a Map Object containing a list of lists
"""
list_without_test_parent = u[:]
list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size))
def ctpc_algorithm(self):
"""
Compute the CTPC algorithm.
@ -289,26 +249,4 @@ class StructureEstimator:
total_vars_numb = self.sample_path.total_variables_count
[ctpc_algo(n, total_vars_numb) for n in self.nodes]
def save_results(self):
"""
Save the estimated Structure to a .json file
Parameters:
void
Returns:
void
"""
res = json_graph.node_link_data(self.complete_graph)
name = self.sample_path.importer.file_path.rsplit('/',1)[-1]
#print(name)
name = 'results_' + name
with open(name, 'w+') as f:
json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)

@ -0,0 +1,96 @@
import sys
sys.path.append('../')
import itertools
import json
import typing
import networkx as nx
import numpy as np
from networkx.readwrite import json_graph
from abc import ABC
import utility.cache as ch
import structure_graph.conditional_intensity_matrix as condim
import structure_graph.network_graph as ng
import estimators.parameters_estimator as pe
import structure_graph.sample_path as sp
import structure_graph.structure as st
class StructureEstimator(ABC):
"""
Has the task of estimating the network structure given the trajectories in samplepath.
:sample_path: the sample_path object containing the trajectories and the real structure
:nodes: the nodes labels
:nodes_vals: the nodes cardinalities
:nodes_indxs: the nodes indexes
:complete_graph: the complete directed graph built using the nodes labels in nodes
:cache: the cache object
"""
def __init__(self, sample_path: sp.SamplePath):
self.sample_path = sample_path
self.nodes = np.array(self.sample_path.structure.nodes_labels)
self.nodes_vals = self.sample_path.structure.nodes_values
self.nodes_indxs = self.sample_path.structure.nodes_indexes
self.complete_graph = self.build_complete_graph(self.sample_path.structure.nodes_labels)
self.cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List):
"""
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
Parameters:
node_ids: the list of nodes labels
Returns:
a complete Digraph Object
"""
complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids)
complete_graph.add_edges_from(itertools.permutations(node_ids, 2))
return complete_graph
def generate_possible_sub_sets_of_size(self, u: typing.List, size: int, parent_label: str):
"""
Creates a list containing all possible subsets of the list u of size size,
that do not contains a the node identified by parent_label.
Parameters:
u: the list of nodes
size: the size of the subsets
parent_label: the nodes to exclude in the subsets generation
Returns:
a Map Object containing a list of lists
"""
list_without_test_parent = u[:]
list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size))
def save_results(self):
"""
Save the estimated Structure to a .json file
Parameters:
void
Returns:
void
"""
res = json_graph.node_link_data(self.complete_graph)
name = self.sample_path.importer.file_path.rsplit('/',1)[-1]
#print(name)
name = '../results_' + name
with open(name, 'w+') as f:
json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)

@ -1,4 +1,5 @@
import sys
sys.path.append('../')
import itertools
import json
import typing
@ -10,15 +11,18 @@ from networkx.readwrite import json_graph
from random import choice
import copy
import cache as ch
import conditional_intensity_matrix as condim
import network_graph as ng
import parameters_estimator as pe
import sample_path as sp
import structure as st
import fam_score_calculator as fam_score
import utility.cache as ch
import structure_graph.conditional_intensity_matrix as condim
import structure_graph.network_graph as ng
import estimators.parameters_estimator as pe
import estimators.structure_estimator as se
import structure_graph.sample_path as sp
import structure_graph.structure as st
import estimators.fam_score_calculator as fam_score
from utility.decorators import timing
from decorators import timing
#from numba import njit
@ -32,41 +36,15 @@ from multiprocessing import Pool
#TODO: Evaluate if it could be better to change list_edges to set for improve the performance
'''
class StructureScoreBasedEstimator:
class StructureScoreBasedEstimator(se.StructureEstimator):
"""
Has the task of estimating the network structure given the trajectories in samplepath by
using a score based approach.
:sample_path: the sample_path object containing the trajectories and the real structure
:nodes: the nodes labels
:nodes_vals: the nodes cardinalities
:nodes_indxs: the nodes indexes
:complete_graph: the complete directed graph built using the nodes labels in nodes
:cache: the cache object
"""
def __init__(self, sample_path: sp.SamplePath):
self.sample_path = sample_path
self.nodes = np.array(self.sample_path.structure.nodes_labels)
self.nodes_vals = self.sample_path.structure.nodes_values
self.nodes_indxs = self.sample_path.structure.nodes_indexes
self.complete_graph = self.build_complete_graph(self.sample_path.structure.nodes_labels)
self.cache = ch.Cache()
def build_complete_graph(self, node_ids: typing.List):
"""
Builds a complete directed graph (no self loops) given the nodes labels in the list node_ids:
Parameters:
node_ids: the list of nodes labels
Returns:
a complete Digraph Object
"""
complete_graph = nx.DiGraph()
complete_graph.add_nodes_from(node_ids)
complete_graph.add_edges_from(itertools.permutations(node_ids, 2))
return complete_graph
super().__init__(sample_path)
@timing
@ -241,26 +219,4 @@ class StructureScoreBasedEstimator:
list_without_test_parent.remove(parent_label)
return map(list, itertools.combinations(list_without_test_parent, size))
def save_results(self):
"""
Save the estimated Structure to a .json file
Parameters:
void
Returns:
void
"""
res = json_graph.node_link_data(self.complete_graph)
name = self.sample_path.importer.file_path.rsplit('/',1)[-1]
#print(name)
name = 'results_' + name
with open(name, 'w+') as f:
json.dump(res, f)
def remove_diagonal_elements(self, matrix):
m = matrix.shape[0]
strided = np.lib.stride_tricks.as_strided
s0, s1 = matrix.strides
return strided(matrix.ravel()[1:], shape=(m - 1, m), strides=(s0 + s1, s1)).reshape(m, -1)

@ -1,6 +1,9 @@
from abc import ABC, abstractmethod
import abstract_importer as ai
import sys
sys.path.append('../')
import utility.abstract_importer as ai
class AbstractSamplePath(ABC):

@ -1,5 +1,7 @@
import numpy as np
import sys
sys.path.append('../')
class ConditionalIntensityMatrix:
"""

@ -4,6 +4,8 @@ import typing
import networkx as nx
import numpy as np
import sys
sys.path.append('../')
class NetworkGraph:
"""

@ -1,7 +1,11 @@
import abstract_sample_path as asam
import json_importer as imp
import structure as st
import trajectory as tr
import sys
sys.path.append('../')
import structure_graph.abstract_sample_path as asam
import utility.json_importer as imp
import structure_graph.structure as st
import structure_graph.trajectory as tr
class SamplePath(asam.AbstractSamplePath):

@ -1,8 +1,11 @@
import sys
sys.path.append('../')
import typing
import numpy as np
import conditional_intensity_matrix as cim
import structure_graph.conditional_intensity_matrix as cim
class SetOfCims:

@ -1,4 +1,7 @@
import set_of_cims as socim
import sys
sys.path.append('../')
import structure_graph.set_of_cims as socim
class SetsOfCimsContainer:

@ -1,3 +1,6 @@
import sys
sys.path.append('../')
import typing as ty
import numpy as np

@ -1,6 +1,9 @@
import sys
sys.path.append('../')
import typing
import set_of_cims as sofc
import structure_graph.set_of_cims as sofc
class Cache:

@ -1,10 +1,12 @@
import sys
sys.path.append('../')
import json
import typing
import pandas as pd
import abstract_importer as ai
import utility.abstract_importer as ai
class JsonImporter(ai.AbstractImporter):

@ -1,22 +1,22 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import glob
import os
import network_graph as ng
import sample_path as sp
import set_of_cims as sofc
import parameters_estimator as pe
import json_importer as ji
import structure_graph.network_graph as ng
import structure_graph.sample_path as sp
import structure_graph.set_of_cims as sofc
import estimators.parameters_estimator as pe
import utility.json_importer as ji
class TestParametersEstimatior(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()

@ -1,8 +1,8 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import structure as st
import structure_graph.structure as st
class TestStructure(unittest.TestCase):

@ -1,5 +1,5 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import glob
import math
import os
@ -10,17 +10,17 @@ import numpy as np
import psutil
from line_profiler import LineProfiler
import cache as ch
import sample_path as sp
import structure_estimator as se
import json_importer as ji
import utility.cache as ch
import structure_graph.sample_path as sp
import estimators.structure_constraint_based_estimator as se
import utility.json_importer as ji
class TestStructureEstimator(unittest.TestCase):
class TestStructureConstraintBasedEstimator(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()
@ -29,7 +29,7 @@ class TestStructureEstimator(unittest.TestCase):
def test_init(self):
exp_alfa = 0.1
chi_alfa = 0.1
se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa)
se1 = se.StructureConstraintBasedEstimator(self.s1, exp_alfa, chi_alfa)
self.assertEqual(self.s1, se1.sample_path)
self.assertTrue(np.array_equal(se1.nodes, np.array(self.s1.structure.nodes_labels)))
self.assertTrue(np.array_equal(se1.nodes_indxs, self.s1.structure.nodes_indexes))
@ -43,7 +43,7 @@ class TestStructureEstimator(unittest.TestCase):
exp_alfa = 0.1
chi_alfa = 0.1
nodes_numb = len(self.s1.structure.nodes_labels)
se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa)
se1 = se.StructureConstraintBasedEstimator(self.s1, exp_alfa, chi_alfa)
cg = se1.build_complete_graph(self.s1.structure.nodes_labels)
self.assertEqual(len(cg.edges), nodes_numb*(nodes_numb - 1))
for node in self.s1.structure.nodes_labels:
@ -56,7 +56,7 @@ class TestStructureEstimator(unittest.TestCase):
exp_alfa = 0.1
chi_alfa = 0.1
nodes_numb = len(self.s1.structure.nodes_labels)
se1 = se.StructureEstimator(self.s1, exp_alfa, chi_alfa)
se1 = se.StructureConstraintBasedEstimator(self.s1, exp_alfa, chi_alfa)
for node in self.s1.structure.nodes_labels:
for b in range(nodes_numb):
@ -68,7 +68,7 @@ class TestStructureEstimator(unittest.TestCase):
self.assertFalse(node in sset)
def test_time(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
se1 = se.StructureConstraintBasedEstimator(self.s1, 0.1, 0.1)
lp = LineProfiler()
lp.add_function(se1.complete_test)
lp.add_function(se1.one_iteration_of_CTPC_algorithm)
@ -89,7 +89,7 @@ class TestStructureEstimator(unittest.TestCase):
se1.save_results()
def test_memory(self):
se1 = se.StructureEstimator(self.s1, 0.1, 0.1)
se1 = se.StructureConstraintBasedEstimator(self.s1, 0.1, 0.1)
se1.ctpc_algorithm()
current_process = psutil.Process(os.getpid())
mem = current_process.memory_info().rss

@ -1,5 +1,5 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import glob
import math
import os
@ -10,10 +10,10 @@ import numpy as np
import psutil
from line_profiler import LineProfiler
import cache as ch
import sample_path as sp
import structure_score_based_estimator as se
import json_importer as ji
import utility.cache as ch
import structure_graph.sample_path as sp
import estimators.structure_score_based_estimator as se
import utility.json_importer as ji
@ -21,7 +21,7 @@ class TestStructureScoreBasedEstimator(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()

@ -1,9 +1,9 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import conditional_intensity_matrix as cim
import structure_graph.conditional_intensity_matrix as cim
class TestConditionalIntensityMatrix(unittest.TestCase):

@ -1,5 +1,5 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import glob
import os
@ -7,16 +7,16 @@ import networkx as nx
import numpy as np
import itertools
import sample_path as sp
import network_graph as ng
import json_importer as ji
import structure_graph.sample_path as sp
import structure_graph.network_graph as ng
import utility.json_importer as ji
class TestNetworkGraph(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
cls.s1 = sp.SamplePath(cls.importer)
cls.s1.build_trajectories()

@ -1,19 +1,19 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import glob
import os
import json_importer as ji
import sample_path as sp
import trajectory as tr
import structure as st
import utility.json_importer as ji
import structure_graph.sample_path as sp
import structure_graph.trajectory as tr
import structure_graph.structure as st
class TestSamplePath(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
cls.importer = ji.JsonImporter(cls.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')
def test_init(self):

@ -1,10 +1,10 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import itertools
import set_of_cims as soci
import structure_graph.set_of_cims as soci
class TestSetOfCims(unittest.TestCase):

@ -1,8 +1,8 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import set_of_cims as sc
import sets_of_cims_container as scc
import structure_graph.set_of_cims as sc
import structure_graph.sets_of_cims_container as scc
class TestSetsOfCimsContainer(unittest.TestCase):

@ -1,9 +1,9 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import trajectory as tr
import structure_graph.trajectory as tr
class TestTrajectory(unittest.TestCase):

@ -1,10 +1,10 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import numpy as np
import cache as ch
import set_of_cims as soci
import utility.cache as ch
import structure_graph.set_of_cims as soci
class TestCache(unittest.TestCase):

@ -1,11 +1,11 @@
import sys
sys.path.append("../classes/")
sys.path.append("../../classes/")
import unittest
import os
import glob
import numpy as np
import pandas as pd
import json_importer as ji
import utility.json_importer as ji
import json
@ -15,7 +15,7 @@ class TestJsonImporter(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.read_files = glob.glob(os.path.join('../data', "*.json"))
cls.read_files = glob.glob(os.path.join('../../data', "*.json"))
def test_init(self):
j1 = ji.JsonImporter(self.read_files[0], 'samples', 'dyn.str', 'variables', 'Time', 'Name')