Skip to content

Commit a670d63

Browse files
JBrisJBris
committed
Adding functional tests for unsupported algorithms
1 parent 3522143 commit a670d63

14 files changed

+900
-14
lines changed

‎calisim/optimisation/emukit_wrapper.py

-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
from emukit.bayesian_optimization.acquisitions.local_penalization import (
1515
LocalPenalization,
1616
)
17-
from emukit.bayesian_optimization.acquisitions.log_acquisition import LogAcquisition
1817
from emukit.bayesian_optimization.loops import BayesianOptimizationLoop
1918
from emukit.core.initial_designs import RandomDesign
2019
from matplotlib import pyplot as plt
@@ -52,7 +51,6 @@ def target_function(X: np.ndarray) -> np.ndarray:
5251
acquisition_funcs = dict(
5352
ei=ExpectedImprovement,
5453
poi=ProbabilityOfImprovement,
55-
la=LogAcquisition,
5654
lp=LocalPenalization,
5755
nlcb=NegativeLowerConfidenceBound,
5856
)

‎calisim/uncertainty/chaospy_wrapper.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ def execute(self) -> None:
5555
solver_name = self.specification.solver
5656
if solver_name not in solvers:
5757
raise ValueError(
58-
f"Unsupported Chaospy solver: {solver_name}.",
59-
f"Supported Chaospy solvers are {', '.join(solvers)}",
58+
f"Unsupported solver: {solver_name}.",
59+
f"Supported solvers are {', '.join(solvers)}",
6060
)
6161

6262
order = self.specification.order

‎calisim/uncertainty/openturns_wrapper.py

+8-6
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,19 @@ def target_function(X: np.ndarray) -> np.ndarray:
4949
self.X_test = None
5050
self.Y_test = None
5151

52+
solvers = ["functional_chaos", "kriging"]
5253
solver_name = self.specification.solver
54+
if solver_name not in solvers:
55+
raise ValueError(
56+
f"Unsupported solver: {solver_name}.",
57+
f"Supported solvers are {', '.join(solvers)}",
58+
)
59+
5360
if solver_name == "functional_chaos":
5461
estimator = FunctionalChaosEstimator(
5562
self.parameters, self.specification.order
5663
)
57-
elif solver_name == "kriging":
64+
else:
5865
method_kwargs = self.specification.method_kwargs
5966
if method_kwargs is None:
6067
method_kwargs = {}
@@ -68,11 +75,6 @@ def target_function(X: np.ndarray) -> np.ndarray:
6875
method_kwargs["parameters"] = self.parameters
6976
method_kwargs["n_out"] = self.specification.n_out
7077
estimator = KrigingEstimator(**method_kwargs)
71-
else:
72-
raise ValueError(
73-
"Solver must be 'functional_chaos' or 'kriging' \
74-
for OpenTurnsUncertaintyAnalysis execute()"
75-
)
7678

7779
estimator.fit(X, Y)
7880
self.emulator = estimator

‎poetry.lock

+18-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ skorch = { version = "^1.0.0", optional = true }
6868
evotorch = {version = "^0.5.1", optional = true}
6969
hydra-core = {version = "^1.3.2", optional = true}
7070
torchx = {extras = ["aws-batch", "kubernetes", "ray"], version = "^0.7.0", optional = true}
71+
cmaes = "^0.11.1"
7172

7273
[tool.poetry.group.docs.dependencies]
7374
sphinx-rtd-theme = "^3.0.1"

‎tests/functional/test_active_learning.py

+72
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
66
"""
77

8+
import pytest
9+
810
from calisim.active_learning import (
911
ActiveLearningMethod,
1012
ActiveLearningMethodModel,
@@ -15,6 +17,76 @@
1517
from ..conftest import get_calibrator
1618

1719

20+
def test_skactiveml_unsupported_surrogate(
21+
sir_model: ExampleModelContainer,
22+
sir_parameter_spec: ParameterSpecification,
23+
outdir: str,
24+
) -> None:
25+
observed_data = sir_model.observed_data
26+
27+
calibration_kwargs = dict(
28+
n_init=20,
29+
n_iterations=10,
30+
n_samples=50,
31+
lr=0.01,
32+
use_shap=True,
33+
method="__functional_test__",
34+
query_strategy="greedy_sampling_target",
35+
calibration_func_kwargs=dict(t=observed_data.day),
36+
method_kwargs=dict(alpha=1e-10, optimizer="fmin_l_bfgs_b"),
37+
)
38+
39+
calibrator = get_calibrator(
40+
ActiveLearningMethod,
41+
ActiveLearningMethodModel,
42+
sir_model,
43+
sir_parameter_spec,
44+
"skactiveml",
45+
outdir,
46+
sir_model.output_labels,
47+
calibration_kwargs,
48+
)
49+
50+
with pytest.raises(ValueError) as exc_info:
51+
calibrator.specify().execute().analyze()
52+
assert "Unsupported surrogate class" in str(exc_info.value)
53+
54+
55+
def test_skactiveml_unsupported_query_strategy(
56+
sir_model: ExampleModelContainer,
57+
sir_parameter_spec: ParameterSpecification,
58+
outdir: str,
59+
) -> None:
60+
observed_data = sir_model.observed_data
61+
62+
calibration_kwargs = dict(
63+
n_init=20,
64+
n_iterations=10,
65+
n_samples=50,
66+
lr=0.01,
67+
use_shap=True,
68+
method="gp",
69+
query_strategy="__functional_test__",
70+
calibration_func_kwargs=dict(t=observed_data.day),
71+
method_kwargs=dict(alpha=1e-10, optimizer="fmin_l_bfgs_b"),
72+
)
73+
74+
calibrator = get_calibrator(
75+
ActiveLearningMethod,
76+
ActiveLearningMethodModel,
77+
sir_model,
78+
sir_parameter_spec,
79+
"skactiveml",
80+
outdir,
81+
sir_model.output_labels,
82+
calibration_kwargs,
83+
)
84+
85+
with pytest.raises(ValueError) as exc_info:
86+
calibrator.specify().execute().analyze()
87+
assert "Unsupported query strategy" in str(exc_info.value)
88+
89+
1890
def test_skactiveml_gp_greedy_sampling_target(
1991
sir_model: ExampleModelContainer,
2092
sir_parameter_spec: ParameterSpecification,

‎tests/functional/test_evolutionary.py

+35
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,41 @@
164164
# assert is_close(sir_model, calibrator)
165165

166166

167+
@pytest.mark.torch
168+
def test_evotorch_unsupported_algorithm(
169+
sir_model: ExampleModelContainer,
170+
sir_parameter_spec: ParameterSpecification,
171+
outdir: str,
172+
l2_norm_metric: DistanceMetricBase,
173+
) -> None:
174+
observed_data = sir_model.observed_data
175+
176+
calibration_kwargs = dict(
177+
n_iterations=10,
178+
n_samples=10,
179+
method="__functional_test__",
180+
directions=["minimize"],
181+
calibration_func_kwargs=dict(t=observed_data.day),
182+
method_kwargs=dict(popsize=100, tournament_size=20, mutation_stdev=1),
183+
)
184+
185+
calibrator = get_calibrator(
186+
EvolutionaryMethod,
187+
EvolutionaryMethodModel,
188+
sir_model,
189+
sir_parameter_spec,
190+
"evotorch",
191+
outdir,
192+
sir_model.output_labels,
193+
calibration_kwargs,
194+
l2_norm_metric,
195+
)
196+
197+
with pytest.raises(ValueError) as exc_info:
198+
calibrator.specify().execute().analyze()
199+
assert "Unsupported EvoTorch algorithm" in str(exc_info.value)
200+
201+
167202
@pytest.mark.torch
168203
def test_evotorch_ga(
169204
sir_model: ExampleModelContainer,

‎tests/functional/test_experimental_design.py

+34
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
66
"""
77

8+
import pytest
9+
810
from calisim.base import ExampleModelContainer
911
from calisim.data_model import ParameterSpecification
1012
from calisim.experimental_design import (
@@ -15,6 +17,38 @@
1517
from ..conftest import get_calibrator
1618

1719

20+
def test_emukit_unsupported_acquisition(
21+
sir_model: ExampleModelContainer,
22+
sir_parameter_spec: ParameterSpecification,
23+
outdir: str,
24+
) -> None:
25+
observed_data = sir_model.observed_data
26+
27+
calibration_kwargs = dict(
28+
n_init=5,
29+
n_iterations=10,
30+
n_samples=50,
31+
method="__functional_test__",
32+
calibration_func_kwargs=dict(t=observed_data.day),
33+
method_kwargs=dict(noise_var=1e-4),
34+
)
35+
36+
calibrator = get_calibrator(
37+
ExperimentalDesignMethod,
38+
ExperimentalDesignMethodModel,
39+
sir_model,
40+
sir_parameter_spec,
41+
"emukit",
42+
outdir,
43+
sir_model.output_labels,
44+
calibration_kwargs,
45+
)
46+
47+
with pytest.raises(ValueError) as exc_info:
48+
calibrator.specify().execute().analyze()
49+
assert "Unsupported emulator acquisition type" in str(exc_info.value)
50+
51+
1852
def test_emukit(
1953
sir_model: ExampleModelContainer,
2054
sir_parameter_spec: ParameterSpecification,

‎tests/functional/test_history_matching.py

+68
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"""
77

88
import numpy as np
9+
import pytest
910

1011
from calisim.base import ExampleModelContainer
1112
from calisim.data_model import ParameterSpecification
@@ -14,6 +15,39 @@
1415
from ..conftest import get_calibrator, is_close
1516

1617

18+
def test_ies_unsupported_smoother(
19+
sir_model: ExampleModelContainer,
20+
sir_parameter_spec: ParameterSpecification,
21+
outdir: str,
22+
) -> None:
23+
observed_data = sir_model.observed_data
24+
output_labels = sir_model.output_labels
25+
26+
calibration_kwargs = dict(
27+
method="__functional_test__",
28+
n_samples=50,
29+
n_iterations=10,
30+
covariance=np.eye(observed_data[output_labels].values.flatten().shape[0]),
31+
calibration_func_kwargs=dict(t=observed_data.day),
32+
method_kwargs=dict(truncation=1.0),
33+
)
34+
35+
calibrator = get_calibrator(
36+
HistoryMatchingMethod,
37+
HistoryMatchingMethodModel,
38+
sir_model,
39+
sir_parameter_spec,
40+
"ies",
41+
outdir,
42+
output_labels,
43+
calibration_kwargs,
44+
)
45+
46+
with pytest.raises(ValueError) as exc_info:
47+
calibrator.specify().execute().analyze()
48+
assert "Unsupported iterative ensemble smoother" in str(exc_info.value)
49+
50+
1751
def test_ies_sies(
1852
sir_model: ExampleModelContainer,
1953
sir_parameter_spec: ParameterSpecification,
@@ -78,6 +112,40 @@ def test_ies_esmda(
78112
assert is_close(sir_model, calibrator)
79113

80114

115+
def test_pyesmda_unsupported_smoother(
116+
sir_model: ExampleModelContainer,
117+
sir_parameter_spec: ParameterSpecification,
118+
outdir: str,
119+
) -> None:
120+
observed_data = sir_model.observed_data
121+
output_labels = sir_model.output_labels
122+
123+
calibration_kwargs = dict(
124+
method="__functional_test__",
125+
n_samples=50,
126+
n_iterations=20,
127+
covariance=np.eye(observed_data[output_labels].values.flatten().shape[0]),
128+
calibration_func_kwargs=dict(t=observed_data.day),
129+
method_kwargs=dict(save_ensembles_history=True),
130+
n_jobs=10,
131+
)
132+
133+
calibrator = get_calibrator(
134+
HistoryMatchingMethod,
135+
HistoryMatchingMethodModel,
136+
sir_model,
137+
sir_parameter_spec,
138+
"pyesmda",
139+
outdir,
140+
output_labels,
141+
calibration_kwargs,
142+
)
143+
144+
with pytest.raises(ValueError) as exc_info:
145+
calibrator.specify().execute().analyze()
146+
assert "Unsupported ensemble smoother" in str(exc_info.value)
147+
148+
81149
def test_pyesmda_esmda_rs(
82150
sir_model: ExampleModelContainer,
83151
sir_parameter_spec: ParameterSpecification,

0 commit comments

Comments
 (0)