From 8bfb27183ec457e9f56b1ffd8fc6da9c33ffdd11 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Tue, 15 Mar 2022 15:22:23 +0100 Subject: [PATCH 1/3] smacsmac2 --- nevergrad/optimization/recastlib.py | 163 +++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 2 deletions(-) diff --git a/nevergrad/optimization/recastlib.py b/nevergrad/optimization/recastlib.py index 9c3cc80c5..93792a033 100644 --- a/nevergrad/optimization/recastlib.py +++ b/nevergrad/optimization/recastlib.py @@ -10,6 +10,7 @@ import weakref import numpy as np from scipy import optimize as scipyoptimize +from unittest import SkipTest import nevergrad.common.typing as tp from nevergrad.parametrization import parameter as p from nevergrad.common import errors @@ -39,13 +40,15 @@ def __init__( "COBYLA", "SLSQP", "NLOPT", + "SMAC", + "SMAC2", "Powell", ], f"Unknown method '{method}'" self.method = method self.random_restart = random_restart # The following line rescales to [0, 1] if fully bounded. - if method in ("CmaFmin2", "NLOPT"): + if method in ("CmaFmin2", "NLOPT", "SMAC", "SMAC2"): normalizer = p.helpers.Normalizer(self.parametrization) if normalizer.fully_bounded: self._normalizer = normalizer @@ -107,7 +110,161 @@ def nlopt_objective_function(*args): # print("With %i function calls" % objfunc_calculator.n_calls) if weakself._normalizer is not None: best_x = weakself._normalizer.backward(np.asarray(best_x, dtype=np.float32)) - + elif weakself.method == "SMAC2": + raise SkipTest("This needs SMAC, which has complicated requirements.") + try: + import smac # noqa # pylint: disable=unused-import + except ImportError as e: + print("You have to install SMAC and its dependencies.") + raise e + + from ConfigSpace.hyperparameters import ( + UniformFloatHyperparameter, + ) # noqa # pylint: disable=unused-import + + # Import ConfigSpace and different types of parameters + from smac.configspace import ConfigurationSpace # noqa # pylint: disable=unused-import + from smac.facade.smac_hpo_facade import SMAC4HPO # noqa # pylint: disable=unused-import + + # Import SMAC-utilities + from smac.scenario.scenario import Scenario # noqa # pylint: disable=unused-import + import threading + import os + import time + from pathlib import Path + the_date = str(time.time()) + feed = "/tmp/smac_feed" + the_date + ".txt" + fed = "/tmp/smac_fed" + the_date + ".txt" + def dummy_function(): + for u in range(remaining): + print(f"side thread waiting for request... ({u}/{weakself.budget})") + while (not Path(feed).is_file()) or os.stat(feed).st_size == 0: + time.sleep(0.1) + time.sleep(0.1) + print("side thread happy to work on a request...") + data = np.loadtxt(feed) + os.remove(feed) + print("side thread happy to really work on a request...") + res = objective_function(data) + print("side thread happy to forward the result of a request...") + f = open(fed, "w") + f.write(str(res)) + f.close() + return + thread = threading.Thread(target=dummy_function) + thread.start() + + + print(f"start SMAC2 optimization with budget {budget} in dimension {weakself.dimension}") + cs = ConfigurationSpace() + cs.add_hyperparameters( + [ + UniformFloatHyperparameter(f"x{i}", 0.0, 1.0, default_value=0.0) + #UniformFloatHyperparameter(f"x{i}", -1.0, 1.0, default_value=0.0) + for i in range(weakself.dimension) + ] + ) + scenario = Scenario( + { + "run_obj": "quality", # we optimize quality (alternatively runtime) + "runcount-limit": budget, # max. number of function evaluations + "cs": cs, # configuration space + "deterministic": "true", + } + ) + def smac2_obj(p): + print(f"SMAC2 proposes {p}") + p = [p[f"x{i}"] for i in range(len(p.keys()))] + data = weakself._normalizer.backward(np.asarray(p, dtype=np.float)) + print(f"converted to {data}") + if Path(fed).is_file(): + os.remove(fed) + np.savetxt(feed, data) + while (not Path(fed).is_file()) or os.stat(fed).st_size == 0: + time.sleep(0.1) + time.sleep(0.1) + f = open(fed, "r") + res = np.float(f.read()) + f.close() + print(f"SMAC2 will receive {res}") + return res + smac = SMAC4HPO(scenario=scenario, rng=weakself._rng.randint(5000), tae_runner=smac2_obj) + res = smac.optimize() + best_x = [res[f"x{k}"] for k in range(len(res.keys()))] + best_x = weakself._normalizer.backward(np.asarray(best_x, dtype=np.float)) + print("end SMAC optimization") + thread.join() + weakself._num_ask = budget + + elif weakself.method == "SMAC": + raise SkipTest("This needs SMAC, which has complicated requirements.") + try: + import smac # noqa # pylint: disable=unused-import + except ImportError as e: + print("You have to install SMAC and its dependencies.") + raise e + import scipy.optimize # noqa # pylint: disable=unused-import + from smac.facade.func_facade import fmin_smac # noqa # pylint: disable=unused-import + + import threading + import os + import time + from pathlib import Path + the_date = str(time.time()) + feed = "/tmp/smac_feed" + the_date + ".txt" + fed = "/tmp/smac_fed" + the_date + ".txt" + def dummy_function(): + for u in range(remaining): + print(f"side thread waiting for request... ({u}/{weakself.budget})") + while (not Path(feed).is_file()) or os.stat(feed).st_size == 0: + time.sleep(0.1) + time.sleep(0.1) + print("side thread happy to work on a request...") + data = np.loadtxt(feed) + os.remove(feed) + print("side thread happy to really work on a request...") + res = objective_function(data) + print("side thread happy to forward the result of a request...") + f = open(fed, "w") + f.write(str(res)) + f.close() + return + thread = threading.Thread(target=dummy_function) + thread.start() + + def smac_obj(p): + print(f"SMAC proposes {p}") + data = weakself._normalizer.backward(np.asarray([p[i] for i in range(len(p))], dtype=np.float)) + print(f"converted to {data}") + if Path(fed).is_file(): + os.remove(fed) + np.savetxt(feed, data) + while (not Path(fed).is_file()) or os.stat(fed).st_size == 0: + time.sleep(0.1) + time.sleep(0.1) + f = open(fed, "r") + res = np.float(f.read()) + f.close() + print(f"SMAC will receive {res}") + return res + + print(f"start SMAC optimization with budget {budget} in dimension {weakself.dimension}") + assert budget is not None + x, cost, _ = fmin_smac( + #func=lambda x: sum([(x_ - 1.234)**2 for x_ in x]), + func=smac_obj, + x0=[0.0] * weakself.dimension, + bounds=[(0, 1)] * weakself.dimension, + maxfun=remaining, + rng=weakself._rng.randint(5000), + ) # Passing a seed makes fmin_smac determistic + print("end SMAC optimization") + thread.join() + weakself._num_ask = budget + + if cost < best_res: + best_res = cost + best_x = weakself._normalizer.backward(np.asarray(x, dtype=np.float)) elif weakself.method == "CmaFmin2": import cma # import inline in order to avoid matplotlib initialization warning @@ -192,6 +349,8 @@ def __init__(self, *, method: str = "Nelder-Mead", random_restart: bool = False) NelderMead = NonObjectOptimizer(method="Nelder-Mead").set_name("NelderMead", register=True) CmaFmin2 = NonObjectOptimizer(method="CmaFmin2").set_name("CmaFmin2", register=True) +SMAC = NonObjectOptimizer(method="SMAC").set_name("SMAC", register=True) +SMAC2 = NonObjectOptimizer(method="SMAC2").set_name("SMAC2", register=True) NLOPT = NonObjectOptimizer(method="NLOPT").set_name("NLOPT", register=True) Powell = NonObjectOptimizer(method="Powell").set_name("Powell", register=True) RPowell = NonObjectOptimizer(method="Powell", random_restart=True).set_name("RPowell", register=True) From 91be37a33bda17845295ec6410a2fd25bd5183b0 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Tue, 15 Mar 2022 15:25:46 +0100 Subject: [PATCH 2/3] fix --- mypy.ini | 2 +- nevergrad/optimization/recastlib.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/mypy.ini b/mypy.ini index 1709efa63..bcfd4354c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ [mypy] -[mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*,olympus.*] +[smac,smac.*,mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*,olympus.*] ignore_missing_imports = True [mypy-nevergrad.functions.rl.agents,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,nlopt,aquacrop.*] diff --git a/nevergrad/optimization/recastlib.py b/nevergrad/optimization/recastlib.py index 93792a033..9a77743d0 100644 --- a/nevergrad/optimization/recastlib.py +++ b/nevergrad/optimization/recastlib.py @@ -132,9 +132,11 @@ def nlopt_objective_function(*args): import os import time from pathlib import Path + the_date = str(time.time()) feed = "/tmp/smac_feed" + the_date + ".txt" fed = "/tmp/smac_fed" + the_date + ".txt" + def dummy_function(): for u in range(remaining): print(f"side thread waiting for request... ({u}/{weakself.budget})") @@ -151,16 +153,16 @@ def dummy_function(): f.write(str(res)) f.close() return + thread = threading.Thread(target=dummy_function) thread.start() - print(f"start SMAC2 optimization with budget {budget} in dimension {weakself.dimension}") cs = ConfigurationSpace() cs.add_hyperparameters( [ UniformFloatHyperparameter(f"x{i}", 0.0, 1.0, default_value=0.0) - #UniformFloatHyperparameter(f"x{i}", -1.0, 1.0, default_value=0.0) + # UniformFloatHyperparameter(f"x{i}", -1.0, 1.0, default_value=0.0) for i in range(weakself.dimension) ] ) @@ -172,6 +174,7 @@ def dummy_function(): "deterministic": "true", } ) + def smac2_obj(p): print(f"SMAC2 proposes {p}") p = [p[f"x{i}"] for i in range(len(p.keys()))] @@ -188,6 +191,7 @@ def smac2_obj(p): f.close() print(f"SMAC2 will receive {res}") return res + smac = SMAC4HPO(scenario=scenario, rng=weakself._rng.randint(5000), tae_runner=smac2_obj) res = smac.optimize() best_x = [res[f"x{k}"] for k in range(len(res.keys()))] @@ -210,9 +214,11 @@ def smac2_obj(p): import os import time from pathlib import Path + the_date = str(time.time()) feed = "/tmp/smac_feed" + the_date + ".txt" fed = "/tmp/smac_fed" + the_date + ".txt" + def dummy_function(): for u in range(remaining): print(f"side thread waiting for request... ({u}/{weakself.budget})") @@ -229,12 +235,15 @@ def dummy_function(): f.write(str(res)) f.close() return + thread = threading.Thread(target=dummy_function) thread.start() def smac_obj(p): print(f"SMAC proposes {p}") - data = weakself._normalizer.backward(np.asarray([p[i] for i in range(len(p))], dtype=np.float)) + data = weakself._normalizer.backward( + np.asarray([p[i] for i in range(len(p))], dtype=np.float) + ) print(f"converted to {data}") if Path(fed).is_file(): os.remove(fed) @@ -251,7 +260,7 @@ def smac_obj(p): print(f"start SMAC optimization with budget {budget} in dimension {weakself.dimension}") assert budget is not None x, cost, _ = fmin_smac( - #func=lambda x: sum([(x_ - 1.234)**2 for x_ in x]), + # func=lambda x: sum([(x_ - 1.234)**2 for x_ in x]), func=smac_obj, x0=[0.0] * weakself.dimension, bounds=[(0, 1)] * weakself.dimension, From eb85deae0cea864f57896c720ba2677bc4d3a46b Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Tue, 15 Mar 2022 15:33:00 +0100 Subject: [PATCH 3/3] fix --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index bcfd4354c..e2e64132c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ [mypy] -[smac,smac.*,mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*,olympus.*] +[mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*,olympus.*,smac,smac.*] ignore_missing_imports = True [mypy-nevergrad.functions.rl.agents,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,nlopt,aquacrop.*]