diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 515439b2b9..9f8b6d9cea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,3 +37,4 @@ repos: rev: v1.19.1 hooks: - id: mypy + exclude: ^libensemble/utils/(launcher|loc_stack|runners|pydantic|output_directory)\.py$|^libensemble/tests/regression_tests/support\.py$|^libensemble/tests/functionality_tests/ diff --git a/docs/data_structures/alloc_specs.rst b/docs/data_structures/alloc_specs.rst index 074c6a1528..159b9eacaf 100644 --- a/docs/data_structures/alloc_specs.rst +++ b/docs/data_structures/alloc_specs.rst @@ -19,7 +19,7 @@ Can be constructed and passed to libEnsemble as a Python class or a dictionary. * libEnsemble uses the following defaults if the user doesn't provide their own ``alloc_specs``: .. literalinclude:: ../../libensemble/specs.py - :start-at: alloc_f: Callable = give_sim_work_first + :start-at: alloc_f: Callable = start_only_persistent :end-before: end_alloc_tag :caption: Default settings for alloc_specs diff --git a/docs/examples/alloc_funcs.rst b/docs/examples/alloc_funcs.rst index f54f4bf3c4..3734d7cb0d 100644 --- a/docs/examples/alloc_funcs.rst +++ b/docs/examples/alloc_funcs.rst @@ -10,14 +10,31 @@ Many users use these unmodified. .. IMPORTANT:: See the API for allocation functions :ref:`here`. + **The default allocation function changed in libEnsemble v2.0 from `give_sim_work_first` to `start_only_persistent `.** + .. note:: - The default allocation function (for non-persistent generators) is :ref:`give_sim_work_first`. - The most commonly used (for persistent generators) is :ref:`start_only_persistent`. + The default allocation function for persistent generators is :ref:`start_only_persistent`. + + The most commonly used allocation function for non-persistent generators is :ref:`give_sim_work_first`. .. role:: underline :class: underline +.. _start_only_persistent_label: + +start_only_persistent +--------------------- +.. automodule:: start_only_persistent + :members: + :undoc-members: + +.. dropdown:: :underline:`start_only_persistent.py` + + .. literalinclude:: ../../libensemble/alloc_funcs/start_only_persistent.py + :language: python + :linenos: + .. _gswf_label: give_sim_work_first @@ -44,20 +61,6 @@ fast_alloc :language: python :linenos: -.. _start_only_persistent_label: - -start_only_persistent ---------------------- -.. automodule:: start_only_persistent - :members: - :undoc-members: - -.. dropdown:: :underline:`start_only_persistent.py` - - .. literalinclude:: ../../libensemble/alloc_funcs/start_only_persistent.py - :language: python - :linenos: - start_persistent_local_opt_gens ------------------------------- .. automodule:: start_persistent_local_opt_gens diff --git a/docs/function_guides/allocator.rst b/docs/function_guides/allocator.rst index a65c404ab9..0620105825 100644 --- a/docs/function_guides/allocator.rst +++ b/docs/function_guides/allocator.rst @@ -128,8 +128,8 @@ The remaining values above are useful for efficient filtering of H values Descriptions of included allocation functions can be found :doc:`here<../examples/alloc_funcs>`. The default allocation function is -``give_sim_work_first``. During its worker ID loop, it checks if there's unallocated +``start_only_persistent``. During its worker ID loop, it checks if there's unallocated work and assigns simulations for that work. Otherwise, it initializes generators for up to ``"num_active_gens"`` instances. Other settings like ``batch_mode`` are also supported. See -:ref:`here` for more information about ``give_sim_work_first``. +:ref:`here` for more information. diff --git a/docs/tutorials/calib_cancel_tutorial.rst b/docs/tutorials/calib_cancel_tutorial.rst index 1c0fcff3c8..c008100d73 100644 --- a/docs/tutorials/calib_cancel_tutorial.rst +++ b/docs/tutorials/calib_cancel_tutorial.rst @@ -151,11 +151,9 @@ The allocation function used in this example is the *only_persistent_gens* funct alloc_specs = { "alloc_f": alloc_f, - "user": { - "init_sample_size": init_sample_size, - "async_return": True, - "active_recv_gen": True, - }, + "initial_batch_size": init_sample_size, + "async_return": True, + "active_recv_gen": True, } **async_return** tells the allocation function to return results to the generator as soon diff --git a/libensemble/alloc_funcs/fast_alloc.py b/libensemble/alloc_funcs/fast_alloc.py index bb009740c8..e2027da1c0 100644 --- a/libensemble/alloc_funcs/fast_alloc.py +++ b/libensemble/alloc_funcs/fast_alloc.py @@ -7,7 +7,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li to evaluate in the simulation function. The fields in ``sim_specs["in"]`` are given. If all entries in `H` have been given a be evaluated, a worker is told to call the generator function, provided this wouldn't result in - more than ``alloc_specs["user"]["num_active_gen"]`` active generators. + more than ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]`` active generators. This fast_alloc variation of give_sim_work_first is useful for cases that simply iterate through H, issuing evaluations in order and, in particular, @@ -23,7 +23,8 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]: return {}, persis_info - user = alloc_specs.get("user", {}) + user = {**gen_specs, **alloc_specs.get("user", {})} + manage_resources = libE_info["use_resource_sets"] support = AllocSupport(W, manage_resources, persis_info, libE_info) diff --git a/libensemble/alloc_funcs/fast_alloc_and_pausing.py b/libensemble/alloc_funcs/fast_alloc_and_pausing.py index f26b5c0780..fd162a6623 100644 --- a/libensemble/alloc_funcs/fast_alloc_and_pausing.py +++ b/libensemble/alloc_funcs/fast_alloc_and_pausing.py @@ -28,13 +28,16 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]: return {}, persis_info + user = {**gen_specs, **alloc_specs.get("user", {})} manage_resources = libE_info["use_resource_sets"] support = AllocSupport(W, manage_resources, persis_info, libE_info) Work = {} gen_count = support.count_gens() if gen_specs["user"].get("single_component_at_a_time"): - assert alloc_specs["user"]["batch_mode"], "Must be in batch mode when using 'single_component_at_a_time'" + assert alloc_specs["user"].get("batch_mode", False) or gen_specs.get( + "batch_mode", False + ), "Must be in batch mode when using 'single_component_at_a_time'" if len(H) != persis_info["H_len"]: # Something new is in the history. persis_info["need_to_give"].update(H["sim_id"][persis_info["H_len"] :].tolist()) @@ -119,13 +122,13 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li break while len(idle_gen_workers): - if gen_count < alloc_specs["user"].get("num_active_gens", gen_count + 1): + if gen_count < user.get("num_active_gens", gen_count + 1): lw = persis_info["last_worker"] last_size = persis_info.get("last_size") if len(H): # Don't give gen instances in batch mode if points are unfinished - if alloc_specs["user"].get("batch_mode") and not all( + if (alloc_specs["user"].get("batch_mode") or gen_specs.get("batch_mode")) and not all( np.logical_or(H["sim_ended"][last_size:], H["paused"][last_size:]) ): break @@ -142,7 +145,7 @@ def give_sim_work_first(W, H, sim_specs, gen_specs, alloc_specs, persis_info, li persis_info["last_worker"] = i persis_info["last_size"] = len(H) - elif gen_count >= alloc_specs["user"].get("num_active_gens", gen_count + 1): + elif gen_count >= user.get("num_active_gens", gen_count + 1): idle_gen_workers = [] return Work, persis_info diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 7ac4d75e5e..11ccf211f0 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -14,18 +14,18 @@ def give_sim_work_first( alloc_specs: dict, persis_info: dict, libE_info: dict, -) -> tuple[dict]: +) -> tuple[dict, dict]: """ Decide what should be given to workers. This allocation function gives any available simulation work first, and only when all simulations are - completed or running does it start (at most ``alloc_specs["user"]["num_active_gens"]``) + completed or running does it start (at most ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]``) generator instances. - Allows for a ``alloc_specs["user"]["batch_mode"]`` where no generation + Allows for a ``gen_specs["batch_mode"]`` or ``alloc_specs["user"]["batch_mode"]`` where no generation work is given out unless all entries in ``H`` are returned. Can give points in highest priority, if ``"priority"`` is a field in ``H``. - If ``alloc_specs["user"]["give_all_with_same_priority"]`` is set to True, then + If ``gen_specs["give_all_with_same_priority"]`` or ``alloc_specs["user"]["give_all_with_same_priority"]`` is set to True, then all points with the same priority value are given as a batch to the sim. Workers performing sims will be assigned resources given in H["resource_sets"] @@ -40,7 +40,7 @@ def give_sim_work_first( `test_uniform_sampling.py `_ # noqa """ - user = alloc_specs.get("user", {}) + user = {**gen_specs, **alloc_specs.get("user", {})} if "cancel_sims_time" in user: # Cancel simulations that are taking too long diff --git a/libensemble/alloc_funcs/persistent_aposmm_alloc.py b/libensemble/alloc_funcs/persistent_aposmm_alloc.py index 3b87d5b5b9..fe3814704c 100644 --- a/libensemble/alloc_funcs/persistent_aposmm_alloc.py +++ b/libensemble/alloc_funcs/persistent_aposmm_alloc.py @@ -21,7 +21,8 @@ def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info if libE_info["sim_max_given"] or not libE_info["any_idle_workers"]: return {}, persis_info - init_sample_size = gen_specs["user"]["initial_sample_size"] + user = {**gen_specs, **alloc_specs.get("user", {})} + init_sample_size = user["initial_batch_size"] manage_resources = libE_info["use_resource_sets"] support = AllocSupport(W, manage_resources, persis_info, libE_info) gen_count = support.count_persis_gens() diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 7781ed3b5f..801dd11802 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -7,12 +7,12 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, libE_info): """ This allocation function will give simulation work if possible, but - otherwise start up to ``alloc_specs["user"]["num_active_gens"]`` + otherwise start up to ``gen_specs["num_active_gens"]`` or ``alloc_specs["user"]["num_active_gens"]`` persistent generators (defaulting to one). By default, evaluation results are given back to the generator once all generated points have been returned from the simulation evaluation. - If ``alloc_specs["user"]["async_return"]`` is set to True, then any + If ``gen_specs["async_return"]`` or ``alloc_specs["user"]["async_return"]`` is set to True, then any returned points are given back to the generator. If any workers are marked as zero_resource_workers, then these will only @@ -56,11 +56,12 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l return {}, persis_info # Initialize alloc_specs["user"] as user. - user = alloc_specs.get("user", {}) + user = {**gen_specs, **alloc_specs.get("user", {})} + manage_resources = libE_info["use_resource_sets"] active_recv_gen = user.get("active_recv_gen", False) # Persistent gen can handle irregular communications - init_sample_size = user.get("init_sample_size", 0) # Always batch return until this many evals complete + initial_batch_size = user.get("initial_batch_size", 0) # Always batch return until this many evals complete batch_give = user.get("give_all_with_same_priority", False) support = AllocSupport(W, manage_resources, persis_info, libE_info) @@ -68,7 +69,7 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l Work = {} # Asynchronous return to generator - async_return = user.get("async_return", False) and sum(H["sim_ended"]) >= init_sample_size + async_return = user.get("async_return", False) and sum(H["sim_ended"]) >= initial_batch_size if gen_count < persis_info.get("num_gens_started", 0): # When a persistent worker is done, trigger a shutdown (returning exit condition of 1) diff --git a/libensemble/gen_funcs/persistent_sampling.py b/libensemble/gen_funcs/persistent_sampling.py index 375d7f4387..6f28ec2caa 100644 --- a/libensemble/gen_funcs/persistent_sampling.py +++ b/libensemble/gen_funcs/persistent_sampling.py @@ -16,9 +16,9 @@ ] -def _get_user_params(user_specs): +def _get_user_params(user_specs, gen_specs): """Extract user params""" - b = user_specs["initial_batch_size"] + b = gen_specs.get("initial_batch_size") or user_specs.get("initial_batch_size") or gen_specs.get("init_sample_size") ub = user_specs["ub"] lb = user_specs["lb"] n = len(lb) # dimension @@ -44,7 +44,7 @@ def persistent_uniform(_, persis_info, gen_specs, libE_info): `test_persistent_uniform_sampling_async.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs.get("user", {}), gen_specs) ps = PersistentSupport(libE_info, EVAL_GEN_TAG) # Send batches until manager sends stop tag @@ -73,7 +73,7 @@ def persistent_uniform_final_update(_, persis_info, gen_specs, libE_info): `test_persistent_uniform_sampling_running_mean.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs.get("user", {}), gen_specs) ps = PersistentSupport(libE_info, EVAL_GEN_TAG) def generate_corners(x, y): @@ -145,7 +145,7 @@ def persistent_request_shutdown(_, persis_info, gen_specs, libE_info): .. seealso:: `test_persistent_uniform_gen_decides_stop.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs.get("user", {}), gen_specs) shutdown_limit = gen_specs["user"]["shutdown_limit"] f_count = 0 ps = PersistentSupport(libE_info, EVAL_GEN_TAG) @@ -173,7 +173,7 @@ def uniform_nonblocking(_, persis_info, gen_specs, libE_info): .. seealso:: `test_persistent_uniform_sampling.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs.get("user", {}), gen_specs) ps = PersistentSupport(libE_info, EVAL_GEN_TAG) # Send batches until manager sends stop tag @@ -223,7 +223,11 @@ def batched_history_matching(_, persis_info, gen_specs, libE_info): lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["initial_batch_size"] + b = ( + gen_specs.get("initial_batch_size") + or gen_specs["user"].get("initial_batch_size") + or gen_specs.get("init_sample_size") + ) q = gen_specs["user"]["num_best_vals"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) @@ -250,7 +254,11 @@ def persistent_uniform_with_cancellations(_, persis_info, gen_specs, libE_info): ub = gen_specs["user"]["ub"] lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["initial_batch_size"] + b = ( + gen_specs.get("initial_batch_size") + or gen_specs["user"].get("initial_batch_size") + or gen_specs.get("init_sample_size") + ) # Start cancelling points from half initial batch onward cancel_from = b // 2 # Should get at least this many points back diff --git a/libensemble/gen_funcs/persistent_sampling_var_resources.py b/libensemble/gen_funcs/persistent_sampling_var_resources.py index 394ef75815..90e14bc4b8 100644 --- a/libensemble/gen_funcs/persistent_sampling_var_resources.py +++ b/libensemble/gen_funcs/persistent_sampling_var_resources.py @@ -25,11 +25,11 @@ ] -def _get_user_params(user_specs): +def _get_user_params(gen_specs): """Extract user params""" - b = user_specs["initial_batch_size"] - ub = user_specs["ub"] - lb = user_specs["lb"] + b = gen_specs["initial_batch_size"] + ub = gen_specs["user"]["ub"] + lb = gen_specs["user"]["lb"] n = len(lb) # dimension return b, n, lb, ub @@ -43,7 +43,7 @@ def uniform_sample(_, persis_info, gen_specs, libE_info): `test_uniform_sampling_with_variable_resources.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) tag = None @@ -76,7 +76,7 @@ def uniform_sample_with_var_gpus(_, persis_info, gen_specs, libE_info): `test_GPU_variable_resources.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) tag = None @@ -111,7 +111,7 @@ def uniform_sample_with_procs_gpus(_, persis_info, gen_specs, libE_info): `test_GPU_variable_resources.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) tag = None @@ -137,7 +137,7 @@ def uniform_sample_with_var_priorities(_, persis_info, gen_specs, libE_info): resource sets and priorities are requested for each point. """ - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) @@ -175,7 +175,7 @@ def uniform_sample_diff_simulations(_, persis_info, gen_specs, libE_info): `test_GPU_variable_resources_multi_task.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) tag = None @@ -209,7 +209,7 @@ def uniform_sample_with_sim_gen_resources(_, persis_info, gen_specs, libE_info): `test_GPU_variable_resources.py `_ """ # noqa - b, n, lb, ub = _get_user_params(gen_specs["user"]) + b, n, lb, ub = _get_user_params(gen_specs) rng = persis_info["rand_stream"] ps = PersistentSupport(libE_info, EVAL_GEN_TAG) tag = None diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index efe9eab407..4a22e0984c 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -19,7 +19,7 @@ @output_data([("x", float, 2)]) # default: can be overwritten in gen_specs def uniform_random_sample(_, persis_info, gen_specs): """ - Generates ``gen_specs["user"]["gen_batch_size"]`` points uniformly over the domain + Generates ``gen_specs["batch_size"]`` points uniformly over the domain defined by ``gen_specs["user"]["ub"]`` and ``gen_specs["user"]["lb"]``. .. seealso:: @@ -29,7 +29,7 @@ def uniform_random_sample(_, persis_info, gen_specs): lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) @@ -40,7 +40,7 @@ def uniform_random_sample(_, persis_info, gen_specs): def uniform_random_sample_with_variable_resources(_, persis_info, gen_specs): """ - Generates ``gen_specs["user"]["gen_batch_size"]`` points uniformly over the domain + Generates ``gen_specs["batch_size"]`` points uniformly over the domain defined by ``gen_specs["user"]["ub"]`` and ``gen_specs["user"]["lb"]``. Also randomly requests a different number of resource sets to be used in each evaluation. @@ -56,7 +56,7 @@ def uniform_random_sample_with_variable_resources(_, persis_info, gen_specs): max_rsets = gen_specs["user"]["max_resource_sets"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) @@ -84,7 +84,7 @@ def uniform_random_sample_with_var_priorities_and_resources(H, persis_info, gen_ n = len(lb) if len(H) == 0: - b = gen_specs["user"]["initial_batch_size"] + b = gen_specs["initial_batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) for i in range(0, b): @@ -119,7 +119,7 @@ def uniform_random_sample_obj_components(H, persis_info, gen_specs): n = len(lb) m = gen_specs["user"]["components"] - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b * m, dtype=gen_specs["out"]) for i in range(0, b): @@ -143,7 +143,7 @@ def uniform_random_sample_cancel(_, persis_info, gen_specs): lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) for i in range(b): @@ -158,7 +158,7 @@ def uniform_random_sample_cancel(_, persis_info, gen_specs): @output_data([("x", float, (1,))]) def latin_hypercube_sample(_, persis_info, gen_specs): """ - Generates ``gen_specs["user"]["gen_batch_size"]`` points in a Latin + Generates ``gen_specs["batch_size"]`` points in a Latin hypercube sample over the domain defined by ``gen_specs["user"]["ub"]`` and ``gen_specs["user"]["lb"]``. @@ -170,7 +170,7 @@ def latin_hypercube_sample(_, persis_info, gen_specs): lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index fca0e70bb6..ea234acff2 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -15,7 +15,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): """ - This generation function returns ``gen_specs["user"]["gen_batch_size"]`` uniformly + This generation function returns ``gen_specs["batch_size"]`` uniformly sampled points when called in nonpersistent mode (i.e., when ``libE_info["persistent"]`` isn't ``True``). Otherwise, the generation function starts a persistent nlopt local optimization run. @@ -31,7 +31,7 @@ def uniform_or_localopt(H, persis_info, gen_specs, libE_info): ub = gen_specs["user"]["ub"] lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) for i in range(0, b): diff --git a/libensemble/specs.py b/libensemble/specs.py index a38efd4cb1..e3835b9cab 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -5,7 +5,7 @@ import pydantic from pydantic import BaseModel, Field, model_validator -from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first +from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens __all__ = ["SimSpecs", "GenSpecs", "AllocSpecs", "ExitCriteria", "LibeSpecs", "_EnsembleSpecs"] @@ -200,20 +200,20 @@ class GenSpecs(BaseModel): initial_batch_size: int = 0 """ - Number of initial points to request that the generator create. If zero, falls back to ``batch_size``. - If both options are zero, defaults to the number of workers. + Initial sample size. + For standardized generators, this is the number of initial points to request that the + generator create. If zero, falls back to ``batch_size``. + For persistent generators, this is the number of points evaluated before switching + from batch return to asynchronous return (if ``async_return`` is True). - Note: Certain generators included with libEnsemble decide - batch sizes via ``gen_specs["user"]`` or other methods. + Note: Certain generators included with libEnsemble decide batch sizes via + ``gen_specs["user"]`` or other methods. """ batch_size: int = 0 """ Number of points to generate in each batch. If zero, falls back to the number of completed evaluations most recently told to the generator. - - Note: Certain generators included with libEnsemble decide - batch sizes via ``gen_specs["user"]`` or other methods. """ threaded: bool | None = False @@ -233,6 +233,44 @@ class GenSpecs(BaseModel): they will be automatically derived from VOCS. """ + # Only used if using the only_persistent_gens allocation function (default) + num_active_gens: int = 1 + """ + Maximum number of persistent generators to start. Default: 1. + Only used if using the ``only_persistent_gens`` allocation function (the default). + """ + + async_return: bool = False + """ + Return results to gen as they come in (after sample). Default: False (batch return). + Only used if using the ``only_persistent_gens`` allocation function (the default). + """ + + active_recv_gen: bool = False + """ + Create gen in active receive mode. If True, the manager does not need to wait + for a return from the generator before sending further returned points. + Default: False. Only used if using the ``only_persistent_gens`` allocation function (the default). + """ + + give_all_with_same_priority: bool = False + """ + If True, then all points with the same priority value are given as a batch to the sim. + Default: False. Only used if using the ``only_persistent_gens`` allocation function (the default). + """ + + alt_type: bool = False + """ + If True, then the specialized allocator behavior for some persistent gens is used. + Only used if using the ``only_persistent_gens`` allocation function (the default). + """ + + batch_mode: bool = False + """ + If True, then the generator will not be started if there are still simulations + running. Only used if using the ``give_sim_work_first`` allocation function. + """ + @model_validator(mode="after") def set_fields_from_vocs(self): """Set persis_in and outputs from VOCS if vocs is provided and fields are not set.""" @@ -280,16 +318,24 @@ class AllocSpecs(BaseModel): Specifications for configuring an Allocation Function. """ - alloc_f: object = give_sim_work_first + alloc_f: object = only_persistent_gens """ Python function matching the ``alloc_f`` interface. Decides when simulator and generator functions should be called, and with what resources and parameters. + + .. note:: + For libEnsemble v2.0, the default allocation function is now ``only_persistent_gens``, instead + of ``give_sim_work_first``. """ - user: dict | None = {"num_active_gens": 1} + user: dict | None = {} """ A user-data dictionary to place bounds, constants, settings, or other parameters for customizing the allocation function. + + .. note:: + As of libEnsemble v2.0, generator-specific allocation options (e.g., ``async_return``, + ``num_active_gens``) have been moved to :class:`GenSpecs`. """ outputs: list[tuple] = Field([], alias="out") diff --git a/libensemble/tests/functionality_tests/sine_gen.py b/libensemble/tests/functionality_tests/sine_gen.py index 38f320e946..88619c0a42 100644 --- a/libensemble/tests/functionality_tests/sine_gen.py +++ b/libensemble/tests/functionality_tests/sine_gen.py @@ -11,7 +11,7 @@ def gen_random_sample(InputArray, persis_info, gen_specs): # Determine how many values to generate num = len(lower) - batch_size = user_specs["gen_batch_size"] + batch_size = gen_specs["batch_size"] # Create empty array of "batch_size" zeros. Array dtype should match "out" fields OutputArray = np.zeros(batch_size, dtype=gen_specs["out"]) diff --git a/libensemble/tests/functionality_tests/test_1d_sampling_no_comms_given.py b/libensemble/tests/functionality_tests/test_1d_sampling_no_comms_given.py index 563ee920e2..676fb3cc83 100644 --- a/libensemble/tests/functionality_tests/test_1d_sampling_no_comms_given.py +++ b/libensemble/tests/functionality_tests/test_1d_sampling_no_comms_given.py @@ -17,11 +17,12 @@ import numpy as np from libensemble import Ensemble +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test from libensemble.sim_funcs.simple_sim import norm_eval as sim_f -from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs from libensemble.tools import check_npy_file_exists # Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). @@ -38,8 +39,8 @@ gen_specs = GenSpecs( gen_f=gen_f, outputs=[("x", float, (1,))], + batch_size=500, user={ - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, @@ -55,6 +56,7 @@ ) sampling.add_random_streams() + sampling.alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) H, persis_info, flag = sampling.run() if sampling.is_manager: diff --git a/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py b/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py index 645228d137..3e83be92f8 100644 --- a/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py +++ b/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py @@ -19,6 +19,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f from libensemble.libE import libE from libensemble.sim_funcs.simple_sim import norm_eval as sim_f @@ -41,8 +42,8 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 500, "user": { - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, @@ -50,10 +51,16 @@ persis_info = add_unique_random_streams({}, nworkers + 1) + alloc_specs = { + "alloc_f": give_sim_work_first, + } + exit_criteria = {"sim_max": 501} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert len(H) >= 501 diff --git a/libensemble/tests/functionality_tests/test_1d_splitcomm.py b/libensemble/tests/functionality_tests/test_1d_splitcomm.py index de73660d70..467afe613a 100644 --- a/libensemble/tests/functionality_tests/test_1d_splitcomm.py +++ b/libensemble/tests/functionality_tests/test_1d_splitcomm.py @@ -15,6 +15,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test @@ -44,19 +45,25 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 500, "user": { - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1, seed=1234) exit_criteria = {"gen_max": 501} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert len(H) >= 501 diff --git a/libensemble/tests/functionality_tests/test_1d_subcomm.py b/libensemble/tests/functionality_tests/test_1d_subcomm.py index 7f607c31c9..cb033527ab 100644 --- a/libensemble/tests/functionality_tests/test_1d_subcomm.py +++ b/libensemble/tests/functionality_tests/test_1d_subcomm.py @@ -15,6 +15,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test @@ -48,19 +49,25 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 500, "user": { - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1, seed=1234) exit_criteria = {"gen_max": 501} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert len(H) >= 501 diff --git a/libensemble/tests/functionality_tests/test_1d_super_simple.py b/libensemble/tests/functionality_tests/test_1d_super_simple.py index e84255714f..67c338e7de 100644 --- a/libensemble/tests/functionality_tests/test_1d_super_simple.py +++ b/libensemble/tests/functionality_tests/test_1d_super_simple.py @@ -15,6 +15,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test @@ -28,10 +29,6 @@ def sim_f(In): return Out -def sim_f_noreturn(In): - print(np.linalg.norm(In)) - - if __name__ == "__main__": nworkers, is_manager, libE_specs, _ = parse_args() @@ -44,8 +41,8 @@ def sim_f_noreturn(In): gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 500, "user": { - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, @@ -55,21 +52,15 @@ def sim_f_noreturn(In): exit_criteria = {"gen_max": 501} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) - - if is_manager: - assert len(H) >= 501 - print("\nlibEnsemble with random sampling has generated enough points") - save_libE_output(H, persis_info, __file__, nworkers) - - # Test running a sim_f without any returns - sim_specs = { - "sim_f": sim_f_noreturn, - "in": ["x"], + alloc_specs = { + "alloc_f": give_sim_work_first, } - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert len(H) >= 501 print("\nlibEnsemble with random sampling has generated enough points") + save_libE_output(H, persis_info, __file__, nworkers) diff --git a/libensemble/tests/functionality_tests/test_1d_uniform_sampling_with_comm_dup.py b/libensemble/tests/functionality_tests/test_1d_uniform_sampling_with_comm_dup.py index 7d2d9f588f..ea070dc72b 100644 --- a/libensemble/tests/functionality_tests/test_1d_uniform_sampling_with_comm_dup.py +++ b/libensemble/tests/functionality_tests/test_1d_uniform_sampling_with_comm_dup.py @@ -19,6 +19,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f # Import libEnsemble items for this test @@ -51,10 +52,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (1,))], + "batch_size": 500, "user": { "lb": np.array([-3]), "ub": np.array([3]), - "gen_batch_size": 500, }, } @@ -62,8 +63,12 @@ exit_criteria = {"gen_max": 501} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs) if is_manager: # assert libE_specs["comms"] == "mpi", "MPI default comms should be set" diff --git a/libensemble/tests/functionality_tests/test_GPU_gen_resources.py b/libensemble/tests/functionality_tests/test_GPU_gen_resources.py index d77088d7e4..8feb36ea77 100644 --- a/libensemble/tests/functionality_tests/test_GPU_gen_resources.py +++ b/libensemble/tests/functionality_tests/test_GPU_gen_resources.py @@ -32,7 +32,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_with_sim_gen_resources as gen_f @@ -79,8 +78,10 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("num_procs", int), ("num_gpus", int), ("x", float, n)], + "initial_batch_size": nworkers - 1, + "give_all_with_same_priority": False, + "async_return": False, "user": { - "initial_batch_size": nworkers - 1, "max_procs": nworkers - 1, # Any sim created can req. 1 worker up to all. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), @@ -88,14 +89,6 @@ }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": False, # False batch returns - }, - } - exit_criteria = {"sim_max": 20} libE_specs["resource_info"] = {"cores_on_node": (nworkers * 2, nworkers * 4), "gpus_on_node": nworkers} @@ -126,8 +119,6 @@ gen_specs["user"]["max_procs"] = max(nworkers - 2, 1) # Perform the run - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # All asserts are in gen and sim funcs diff --git a/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py b/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py index 7d99968629..07c6db8a39 100644 --- a/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py +++ b/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py @@ -48,12 +48,14 @@ "gen_f": gen_f, "persis_in": ["x", "f"], "out": gen_out, + "batch_size": 2, + "batch_mode": True, + "num_active_gens": 1, "user": { "localopt_method": "LN_BOBYQA", "xtol_rel": 1e-4, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": 2, "dist_to_bound_multiple": 0.5, "localopt_maxeval": 4, }, @@ -61,10 +63,6 @@ alloc_specs = { "alloc_f": alloc_f, - "user": { - "batch_mode": True, - "num_active_gens": 1, - }, } persis_info = add_unique_random_streams({}, nworkers + 1) diff --git a/libensemble/tests/functionality_tests/test_asktell_sampling.py b/libensemble/tests/functionality_tests/test_asktell_sampling.py index cebb858df2..fccb9cd6be 100644 --- a/libensemble/tests/functionality_tests/test_asktell_sampling.py +++ b/libensemble/tests/functionality_tests/test_asktell_sampling.py @@ -19,7 +19,6 @@ import libensemble.sim_funcs.six_hump_camel as six_hump_camel # Import libEnsemble items for this test -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_classes.sampling import UniformSample from libensemble.libE import libE from libensemble.sim_funcs.executor_hworld import executor_hworld as sim_f_exec @@ -48,7 +47,6 @@ def sim_f(In): "initial_batch_size": 20, "batch_size": 10, "user": { - "initial_batch_size": 20, # for wrapper "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, @@ -59,7 +57,6 @@ def sim_f(In): vocs = VOCS(variables=variables, objectives=objectives) - alloc_specs = {"alloc_f": alloc_f} exit_criteria = {"gen_max": 201} persis_info = add_unique_random_streams({}, nworkers + 1, seed=1234) @@ -89,9 +86,7 @@ def sim_f(In): } gen_specs["generator"] = generator - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: print(H[["sim_id", "x", "f"]][:10]) diff --git a/libensemble/tests/functionality_tests/test_asktell_sampling_external_gen.py b/libensemble/tests/functionality_tests/test_asktell_sampling_external_gen.py index 8578da720c..318cb49ecf 100644 --- a/libensemble/tests/functionality_tests/test_asktell_sampling_external_gen.py +++ b/libensemble/tests/functionality_tests/test_asktell_sampling_external_gen.py @@ -20,12 +20,12 @@ from libensemble import Ensemble -# Import libEnsemble items for this test -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f - # from libensemble.gen_classes.external.sampling import UniformSampleArray from libensemble.gen_classes.external.sampling import UniformSample -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs + +# Import libEnsemble items for this test + # from gest_api.vocs import ContinuousVariable @@ -75,7 +75,6 @@ def sim_f_scalar(In): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(gen_max=201) ensemble = Ensemble( @@ -83,7 +82,6 @@ def sim_f_scalar(In): sim_specs=sim_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, - alloc_specs=alloc_specs, libE_specs=libE_specs, ) diff --git a/libensemble/tests/functionality_tests/test_calc_exception.py b/libensemble/tests/functionality_tests/test_calc_exception.py index 361e1be3a8..a897f35eb7 100644 --- a/libensemble/tests/functionality_tests/test_calc_exception.py +++ b/libensemble/tests/functionality_tests/test_calc_exception.py @@ -13,6 +13,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.manager import LoggedException @@ -38,15 +39,19 @@ def six_hump_camel_err(H, persis_info, sim_specs, _): "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, 2)], + "batch_size": 10, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": 10, }, } persis_info = add_unique_random_streams({}, nworkers + 1) + alloc_specs = { + "alloc_f": give_sim_work_first, + } + exit_criteria = {"wallclock_max": 10} libE_specs["abort_on_exception"] = False @@ -54,7 +59,9 @@ def six_hump_camel_err(H, persis_info, sim_specs, _): # Perform the run return_flag = 1 try: - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) except LoggedException as e: print(f"Caught deliberate exception: {e}") return_flag = 0 diff --git a/libensemble/tests/functionality_tests/test_cancel_in_alloc.py b/libensemble/tests/functionality_tests/test_cancel_in_alloc.py index d2c005a040..29d3d7cdb1 100644 --- a/libensemble/tests/functionality_tests/test_cancel_in_alloc.py +++ b/libensemble/tests/functionality_tests/test_cancel_in_alloc.py @@ -43,8 +43,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": 5, + "batch_mode": False, + "num_active_gens": 1, "user": { - "gen_batch_size": 5, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, @@ -54,8 +56,6 @@ "alloc_f": give_sim_work_first, "user": { "cancel_sims_time": 3, - "batch_mode": False, - "num_active_gens": 1, }, } diff --git a/libensemble/tests/functionality_tests/test_comms.py b/libensemble/tests/functionality_tests/test_comms.py index 52c3e0771a..7bec22b576 100644 --- a/libensemble/tests/functionality_tests/test_comms.py +++ b/libensemble/tests/functionality_tests/test_comms.py @@ -16,6 +16,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor # Only used to get workerID in float_x1000 from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f @@ -44,10 +45,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": sim_max, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": sim_max, }, } @@ -55,8 +56,14 @@ exit_criteria = {"sim_max": sim_max, "wallclock_max": 300} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert flag == 0 diff --git a/libensemble/tests/functionality_tests/test_elapsed_time_abort.py b/libensemble/tests/functionality_tests/test_elapsed_time_abort.py index 9e7ab97049..feff67c9e8 100644 --- a/libensemble/tests/functionality_tests/test_elapsed_time_abort.py +++ b/libensemble/tests/functionality_tests/test_elapsed_time_abort.py @@ -38,8 +38,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": 5, + "batch_mode": False, + "num_active_gens": 2, "user": { - "gen_batch_size": 5, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, @@ -47,10 +49,6 @@ alloc_specs = { "alloc_f": give_sim_work_first, - "user": { - "batch_mode": False, - "num_active_gens": 2, - }, } persis_info = add_unique_random_streams({}, nworkers + 1) diff --git a/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py b/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py index fe3d8dad8e..b470ccb5a5 100644 --- a/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py +++ b/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py @@ -18,9 +18,10 @@ # Import libEnsemble items for this test from libensemble import Ensemble +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f -from libensemble.specs import ExitCriteria, GenSpecs, SimSpecs +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, SimSpecs from libensemble.tools import add_unique_random_streams @@ -49,8 +50,8 @@ def create_H0(persis_info, gen_specs, H0_size): gen_specs = { "gen_f": gen_f, "outputs": [("x", float, (2,))], + "batch_size": 50, "user": { - "gen_batch_size": 50, "lb": np.array([-3, -3]), "ub": np.array([3, 3]), }, @@ -59,6 +60,7 @@ def create_H0(persis_info, gen_specs, H0_size): sampling.exit_criteria = ExitCriteria(sim_max=100) sampling.persis_info = add_unique_random_streams({}, sampling.nworkers + 1) sampling.H0 = create_H0(sampling.persis_info, gen_specs, 50) + sampling.alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) sampling.run() if sampling.is_manager: diff --git a/libensemble/tests/functionality_tests/test_executor_forces_tutorial.py b/libensemble/tests/functionality_tests/test_executor_forces_tutorial.py index c25ee9e4dc..64dae1727a 100644 --- a/libensemble/tests/functionality_tests/test_executor_forces_tutorial.py +++ b/libensemble/tests/functionality_tests/test_executor_forces_tutorial.py @@ -5,10 +5,9 @@ from forces_simf import run_forces # Sim func from current dir from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs if __name__ == "__main__": # Initialize MPI Executor @@ -43,20 +42,15 @@ inputs=[], # No input when starting persistent generator persis_in=["sim_id"], # Return sim_ids of evaluated points to generator outputs=[("x", float, (1,))], + initial_batch_size=nsim_workers, + async_return=False, user={ - "initial_batch_size": nsim_workers, "lb": np.array([1000]), # min particles "ub": np.array([3000]), # max particles }, ) # gen_specs_end_tag # Starts one persistent generator. Simulated values are returned in batch. - ensemble.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "async_return": False, # False causes batch returns - }, - ) # Instruct libEnsemble to exit after this many simulations ensemble.exit_criteria = ExitCriteria(sim_max=8) diff --git a/libensemble/tests/functionality_tests/test_executor_forces_tutorial_2.py b/libensemble/tests/functionality_tests/test_executor_forces_tutorial_2.py index e3c24fc4f6..5b5fe0a67e 100644 --- a/libensemble/tests/functionality_tests/test_executor_forces_tutorial_2.py +++ b/libensemble/tests/functionality_tests/test_executor_forces_tutorial_2.py @@ -5,10 +5,9 @@ from forces_simf import run_forces # Sim func from current dir from libensemble import Ensemble, logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs logger.set_level("DEBUG") @@ -45,20 +44,15 @@ inputs=[], # No input when starting persistent generator persis_in=["sim_id"], # Return sim_ids of evaluated points to generator outputs=[("x", float, (1,))], + initial_batch_size=nsim_workers, + async_return=True, user={ - "initial_batch_size": nsim_workers, "lb": np.array([1000]), # min particles "ub": np.array([3000]), # max particles }, ) # gen_specs_end_tag # Starts one persistent generator. Simulated values are returned in batch. - ensemble.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "async_return": True, - }, - ) # Instruct libEnsemble to exit after this many simulations ensemble.exit_criteria = ExitCriteria(sim_max=8) diff --git a/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py b/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py index 1b61bf8d25..7266953c36 100644 --- a/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py +++ b/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py @@ -15,6 +15,7 @@ import numpy as np import libensemble.sim_funcs.six_hump_camel as six_hump_camel +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -76,10 +77,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": nworkers, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": nworkers, }, } @@ -88,8 +89,12 @@ # num sim_ended_count conditions in executor_hworld exit_criteria = {"sim_max": nworkers * 5} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) if is_manager: print("\nChecking expected task status against Workers ...\n") diff --git a/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py b/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py index 496eced316..482641eda2 100644 --- a/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py +++ b/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py @@ -15,6 +15,7 @@ import numpy as np import libensemble.sim_funcs.six_hump_camel as six_hump_camel +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f @@ -78,13 +79,17 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": nworkers, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": nworkers, }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"wallclock_max": 10} @@ -97,7 +102,9 @@ for i in range(iterations): # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs + ) if is_manager: print("\nChecking expected task status against Workers ...\n") diff --git a/libensemble/tests/functionality_tests/test_executor_simple.py b/libensemble/tests/functionality_tests/test_executor_simple.py index ac4c201a7a..22ee48fec5 100644 --- a/libensemble/tests/functionality_tests/test_executor_simple.py +++ b/libensemble/tests/functionality_tests/test_executor_simple.py @@ -12,6 +12,7 @@ import numpy as np import libensemble.sim_funcs.six_hump_camel as six_hump_camel +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -49,10 +50,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": nworkers, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": nworkers, }, } @@ -61,8 +62,12 @@ # num sim_ended_count conditions in executor_hworld exit_criteria = {"sim_max": nworkers * 5} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) if is_manager: print("\nChecking expected task status against Workers ...\n") diff --git a/libensemble/tests/functionality_tests/test_fast_alloc.py b/libensemble/tests/functionality_tests/test_fast_alloc.py index a2b6a75666..af4f907c8d 100644 --- a/libensemble/tests/functionality_tests/test_fast_alloc.py +++ b/libensemble/tests/functionality_tests/test_fast_alloc.py @@ -43,8 +43,8 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": num_pts, "user": { - "gen_batch_size": num_pts, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, @@ -73,7 +73,7 @@ if time == 0: sim_specs["user"].pop("uniform_random_pause_ub") - gen_specs["user"]["gen_batch_size"] = num_pts // 2 + gen_specs["batch_size"] = num_pts // 2 persis_info["next_to_give"] = 0 persis_info["total_gen_calls"] = 1 diff --git a/libensemble/tests/functionality_tests/test_local_sine_tutorial.py b/libensemble/tests/functionality_tests/test_local_sine_tutorial.py index 89c3a5e7c8..b09b7c86ed 100644 --- a/libensemble/tests/functionality_tests/test_local_sine_tutorial.py +++ b/libensemble/tests/functionality_tests/test_local_sine_tutorial.py @@ -4,8 +4,7 @@ from sine_sim import sim_find_sine from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs if __name__ == "__main__": # Python-quirk required on macOS and windows libE_specs = LibeSpecs(nworkers=4, comms="local") @@ -20,9 +19,6 @@ batch_size=4, ) - # Specify that libEnsemble should pass work back-and-forth between the generator object - alloc_specs = AllocSpecs(alloc_f=only_persistent_gens) - sim_specs = SimSpecs( sim_f=sim_find_sine, # Our simulator function inputs=["x"], # InputArray field names. "x" from gen_f output @@ -31,7 +27,7 @@ exit_criteria = ExitCriteria(sim_max=80) # Stop libEnsemble after 80 simulations - ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, libE_specs, alloc_specs) + ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, libE_specs) ensemble.add_random_streams() # setup the random streams unique to each worker ensemble.run() # start the ensemble. Blocks until completion. @@ -44,7 +40,7 @@ colors = ["b", "g", "r", "y", "m", "c", "k", "w"] - for i in range(1, libE_specs.nworkers + 1): + for i in range(1, libE_specs.nworkers + 1): # type: ignore worker_xy = np.extract(history["sim_worker"] == i, history) x = [entry.tolist() for entry in worker_xy["x"]] y = [entry for entry in worker_xy["y"]] diff --git a/libensemble/tests/functionality_tests/test_local_sine_tutorial_2.py b/libensemble/tests/functionality_tests/test_local_sine_tutorial_2.py index 11911f343c..ddb61be255 100644 --- a/libensemble/tests/functionality_tests/test_local_sine_tutorial_2.py +++ b/libensemble/tests/functionality_tests/test_local_sine_tutorial_2.py @@ -3,7 +3,8 @@ from sine_sim import sim_find_sine from libensemble import Ensemble -from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs if __name__ == "__main__": libE_specs = LibeSpecs(nworkers=4, comms="local") @@ -11,10 +12,10 @@ gen_specs = GenSpecs( gen_f=gen_random_sample, # Our generator function out=[("x", float, (1,))], # gen_f output (name, type, size) + batch_size=10, # number of x's gen_f generates per call user={ "lower": np.array([-6]), # lower boundary for random sampling "upper": np.array([6]), # upper boundary for random sampling - "gen_batch_size": 10, # number of x's gen_f generates per call }, ) @@ -24,9 +25,11 @@ out=[("y", float)], # sim_f output. "y" = sine("x") ) + alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) + exit_criteria = ExitCriteria(gen_max=160) - ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, libE_specs) + ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, libE_specs, alloc_specs) ensemble.add_random_streams() ensemble.run() diff --git a/libensemble/tests/functionality_tests/test_local_sine_tutorial_3.py b/libensemble/tests/functionality_tests/test_local_sine_tutorial_3.py index d57a0f842b..d67a544331 100644 --- a/libensemble/tests/functionality_tests/test_local_sine_tutorial_3.py +++ b/libensemble/tests/functionality_tests/test_local_sine_tutorial_3.py @@ -3,7 +3,8 @@ from sine_sim import sim_find_sine from libensemble import Ensemble -from libensemble.specs import ExitCriteria, GenSpecs, SimSpecs +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, SimSpecs if __name__ == "__main__": # Python-quirk required on macOS and windows # libE_specs = LibeSpecs(nworkers=4, comms="local") @@ -11,10 +12,10 @@ gen_specs = GenSpecs( gen_f=gen_random_sample, # Our generator function out=[("x", float, (1,))], # gen_f output (name, type, size) + batch_size=5, # number of x's gen_f generates per call user={ "lower": np.array([-3]), # lower boundary for random sampling "upper": np.array([3]), # upper boundary for random sampling - "gen_batch_size": 5, # number of x's gen_f generates per call }, ) @@ -26,8 +27,10 @@ exit_criteria = ExitCriteria(sim_max=80) # Stop libEnsemble after 80 simulations + alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) + # replace libE_specs with parse_args=True. Detects MPI runtime - ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, parse_args=True) + ensemble = Ensemble(sim_specs, gen_specs, exit_criteria, alloc_specs=alloc_specs, parse_args=True) ensemble.add_random_streams() ensemble.run() # start the ensemble. Blocks until completion. diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py index 31e537a31d..203a1ca459 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py @@ -47,7 +47,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample as gen_f @@ -87,22 +86,16 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n)], + "give_all_with_same_priority": False, + "async_return": False, + "initial_batch_size": nworkers - 1, "user": { - "initial_batch_size": nworkers - 1, "max_resource_sets": nworkers - 1, # Any sim created can req. 1 worker up to all. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": False, # False batch returns - }, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 20} @@ -122,9 +115,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["resource_info"] # this would override @@ -159,9 +150,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["platform_specs"] @@ -196,9 +185,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["platform_specs"] @@ -214,9 +201,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["platform"] @@ -232,9 +217,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del os.environ["LIBE_PLATFORM"] @@ -250,9 +233,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["platform_specs"] diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py index 814f5086cb..30c737cbfe 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py @@ -26,7 +26,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample as gen_f @@ -69,22 +68,16 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n)], + "initial_batch_size": nworkers - 1, + "give_all_with_same_priority": False, + "async_return": False, "user": { - "initial_batch_size": nworkers - 1, "max_resource_sets": nworkers - 1, # Any sim created can req. 1 worker up to all. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": False, # False batch returns - }, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 10} @@ -98,4 +91,4 @@ exctr.register_app(full_path=six_hump_camel_app, app_name="six_hump_camel") # Perform the run - H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py index 3c884c7dd2..1234c64821 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py @@ -28,7 +28,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_diff_simulations as gen_f @@ -68,8 +67,10 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("priority", float), ("num_procs", int), ("num_gpus", int), ("x", float, n)], + "initial_batch_size": nsim_workers, + "give_all_with_same_priority": False, + "async_return": False, "user": { - "initial_batch_size": nsim_workers, "max_procs": max(nsim_workers // 2, 1), # Any sim created can req. 1 worker up to max "lb": np.array([-3, -2]), "ub": np.array([3, 2]), @@ -77,14 +78,6 @@ }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": False, # False batch returns - }, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": nsim_workers * 2} @@ -109,9 +102,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # Oversubscribe procs if nsim_workers >= 4: @@ -134,9 +125,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, _, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, _, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) del libE_specs["resource_info"] diff --git a/libensemble/tests/functionality_tests/test_mpi_runners.py b/libensemble/tests/functionality_tests/test_mpi_runners.py index af00471d59..c7d1c88134 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners.py @@ -12,6 +12,7 @@ import numpy as np from libensemble import logger +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -73,10 +74,10 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("x", float, (2,))], + "batch_size": 100, "user": { "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": 100, }, } @@ -233,8 +234,10 @@ def run_tests(mpi_runner, runner_name, test_list_exargs, exp_list): "tests": test_list, } + alloc_specs = {"alloc_f": give_sim_work_first} + # Perform the run - H, pinfo, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, pinfo, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) # for run_set in ['mpich', 'openmpi', 'aprun', 'srun', 'jsrun', 'rename_mpich', 'custom']: for run_set in ["mpich", "aprun", "srun", "jsrun", "rename_mpich", "custom"]: diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py b/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py index 7266678ff2..73d283e9b1 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py @@ -16,6 +16,7 @@ import numpy as np from libensemble import logger +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -84,13 +85,15 @@ "gen_f": gen_f, "in": [], "out": [("x", float, (n,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } + alloc_specs = {"alloc_f": give_sim_work_first} + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": (nsim_workers) * rounds} @@ -118,6 +121,6 @@ } # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py index a5145965b9..26844bcc1d 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py @@ -14,6 +14,7 @@ import numpy as np from libensemble import logger +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -85,13 +86,15 @@ "gen_f": gen_f, "in": [], "out": [("x", float, (n,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } + alloc_specs = {"alloc_f": give_sim_work_first} + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": (nsim_workers) * rounds} @@ -137,6 +140,6 @@ } # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py index 77975e200d..14047d0cb3 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py @@ -11,6 +11,7 @@ import numpy as np from libensemble import logger +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE @@ -75,8 +76,8 @@ "gen_f": gen_f, "in": [], "out": [("x", float, (n,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, @@ -131,7 +132,9 @@ "expect": exp_list, } + alloc_specs = {"alloc_f": give_sim_work_first} + # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py index cc73d0e427..aa2a1a8ebe 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py @@ -21,7 +21,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f from libensemble.libE import libE @@ -100,7 +99,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} exit_criteria = {"sim_max": (nsim_workers) * rounds} test_list_base = [ @@ -161,5 +159,5 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # Run-line asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py index 640d613bff..f52aced21c 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py @@ -11,7 +11,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f from libensemble.libE import libE @@ -86,7 +85,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": (nsim_workers) * rounds} @@ -139,6 +137,6 @@ } # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_mpi_warning.py b/libensemble/tests/functionality_tests/test_mpi_warning.py index daf6125b62..1436e81a02 100644 --- a/libensemble/tests/functionality_tests/test_mpi_warning.py +++ b/libensemble/tests/functionality_tests/test_mpi_warning.py @@ -17,11 +17,12 @@ import numpy as np from libensemble import Ensemble, logger +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test from libensemble.sim_funcs.simple_sim import norm_eval as sim_f -from libensemble.specs import ExitCriteria, GenSpecs, SimSpecs +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, SimSpecs # Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). if __name__ == "__main__": @@ -35,12 +36,13 @@ sampling.gen_specs = GenSpecs( gen_f=gen_f, outputs=[("x", float, 2)], + batch_size=100, user={ - "gen_batch_size": 100, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, ) + sampling.alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) sampling.exit_criteria = ExitCriteria(sim_max=100) sampling.add_random_streams() diff --git a/libensemble/tests/functionality_tests/test_new_field.py b/libensemble/tests/functionality_tests/test_new_field.py index c130b1d6e3..bc5dd4d554 100644 --- a/libensemble/tests/functionality_tests/test_new_field.py +++ b/libensemble/tests/functionality_tests/test_new_field.py @@ -15,6 +15,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test @@ -41,18 +42,24 @@ def sim_f(In): gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 500, "user": { - "gen_batch_size": 500, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1, seed=1234) exit_criteria = {"gen_max": 501} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert len(H) >= 501 diff --git a/libensemble/tests/functionality_tests/test_persistent_sampling_CUDA_variable_resources.py b/libensemble/tests/functionality_tests/test_persistent_sampling_CUDA_variable_resources.py index 327cff0d7d..d80818920a 100644 --- a/libensemble/tests/functionality_tests/test_persistent_sampling_CUDA_variable_resources.py +++ b/libensemble/tests/functionality_tests/test_persistent_sampling_CUDA_variable_resources.py @@ -17,7 +17,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample as gen_f @@ -62,22 +61,16 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("resource_sets", int), ("x", float, n)], + "initial_batch_size": nworkers - 1, + "give_all_with_same_priority": False, + "async_return": True, "user": { - "initial_batch_size": nworkers - 1, "max_resource_sets": nworkers - 1, # Any sim created can req. 1 worker up to all. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": True, - }, - } - libE_specs["scheduler_opts"] = {"match_slots": True} persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 40, "wallclock_max": 300} @@ -87,9 +80,7 @@ for i in range(2): persis_info = add_unique_random_streams({}, nworkers + 1) libE_specs["workflow_dir_path"] = libE_specs["workflow_dir_path"][:-1] + str(i) - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert flag == 0 diff --git a/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py b/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py index f1f0616663..af2a7ac075 100644 --- a/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py @@ -21,7 +21,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f # Import libEnsemble items for this test @@ -57,21 +56,20 @@ "in": [], "persis_in": ["sim_id", "f", "grad"], "out": [("x", float, (n,))], + "initial_batch_size": 5, + "alt_type": True, "user": { - "initial_batch_size": 5, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = {"alloc_f": alloc_f, "user": {"alt_type": True}} - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 40, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert len(np.unique(H["gen_ended_time"])) == 8 diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py index d9b9465080..d9e8594f2e 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_request_shutdown as gen_f # Import libEnsemble items for this test @@ -50,28 +49,22 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("x", float, (n,))], + "initial_batch_size": init_batch_size, + "async_return": True, + "num_active_gens": ngens, "user": { - "initial_batch_size": init_batch_size, "shutdown_limit": 10, # Iterations on a gen before it triggers a shutdown. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "async_return": True, - "num_active_gens": ngens, - }, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"gen_max": 50, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: [ended_times, counts] = np.unique(H["gen_ended_time"], return_counts=True) diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py index 81a18a5285..50c9fd9ce4 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import batched_history_matching as gen_f2 from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f1 @@ -56,8 +55,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} - exit_criteria = {"gen_max": num_batches * batch, "wallclock_max": 300} libE_specs["kill_canceled_sims"] = False @@ -93,7 +90,7 @@ libE_specs["gen_workers"] = [2] # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert len(np.unique(H["gen_ended_time"])) == num_batches diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py index 77d9a6a7b8..3c3d41c6ec 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f # Import libEnsemble items for this test @@ -47,24 +46,20 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("x", float, (n,))], + "initial_batch_size": nworkers, + "async_return": True, "user": { - "initial_batch_size": nworkers, # Ensure > 1 alloc to send all sims "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": {"async_return": True}, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"gen_max": 100, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: [_, counts] = np.unique(H["gen_ended_time"], return_counts=True) diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py index 15c2d5b097..4eec437c3f 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_uniform_with_cancellations as gen_f # Import libEnsemble items for this test @@ -47,24 +46,20 @@ "gen_f": gen_f, "persis_in": ["x", "f", "grad", "sim_id"], "out": [("x", float, (n,))], + "initial_batch_size": 100, + "async_return": True, "user": { - "initial_batch_size": 100, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": {"async_return": True}, - } - exit_criteria = {"gen_max": 150, "wallclock_max": 300} persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: # For reproducible test, only tests if cancel requested on points - not whether got evaluated diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py index 5425578849..c9e92a6c4f 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import uniform_nonblocking as gen_f # Import libEnsemble items for this test @@ -53,8 +52,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} - persis_info = add_unique_random_streams({}, nworkers + 1) for i in persis_info: persis_info[i]["get_grad"] = True @@ -62,7 +59,7 @@ exit_criteria = {"gen_max": 40, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert len(np.unique(H["gen_ended_time"])) == 2 diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py index 9cc92dadd3..9c915c82a4 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py @@ -20,7 +20,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_uniform_final_update as gen_f from libensemble.libE import libE from libensemble.sim_funcs.six_hump_camel import six_hump_camel_simple as sim_f @@ -57,8 +56,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} - sim_max = 120 exit_criteria = {"sim_max": sim_max} libE_specs["final_gen_send"] = True @@ -72,7 +69,7 @@ } persis_info = add_unique_random_streams({}, nworkers + 1) - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: # Check that last saved history agrees with returned history. diff --git a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers.py b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers.py index 34dbb02224..cbff4e2792 100644 --- a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers.py +++ b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers.py @@ -51,8 +51,11 @@ "gen_f": gen_f, "in": ["sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n), ("x_on_cube", float, n)], + "batch_mode": False, + "give_all_with_same_priority": True, + "num_active_gens": 1, + "initial_batch_size": 5, "user": { - "initial_batch_size": 5, "max_resource_sets": 4, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), @@ -61,11 +64,6 @@ alloc_specs = { "alloc_f": give_sim_work_first, - "user": { - "batch_mode": False, - "give_all_with_same_priority": True, - "num_active_gens": 1, - }, } comms = libE_specs["comms"] diff --git a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent.py b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent.py index ab5aa144de..8f80ddb67d 100644 --- a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent.py +++ b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent.py @@ -17,7 +17,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_with_var_priorities as gen_f @@ -61,19 +60,15 @@ "gen_f": gen_f, "persis_in": ["x", "f", "sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n), ("x_on_cube", float, n)], + "initial_batch_size": nworkers - 1, + "give_all_with_same_priority": False, "user": { - "initial_batch_size": nworkers - 1, "max_resource_sets": max_rsets, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": {"give_all_with_same_priority": False}, - } - comms = libE_specs["comms"] node_file = "nodelist_adaptive_workers_persistent_comms_" + str(comms) + "_wrks_" + str(nworkers) if is_manager: @@ -91,9 +86,7 @@ exit_criteria = {"sim_max": 40, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert flag == 0 diff --git a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py index fb730b966b..a317d90433 100644 --- a/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py +++ b/libensemble/tests/functionality_tests/test_runlines_adaptive_workers_persistent_oversubscribe_rsets.py @@ -17,7 +17,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_with_var_priorities as gen_f @@ -63,19 +62,15 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n), ("x_on_cube", float, n)], + "initial_batch_size": nworkers - 1, + "give_all_with_same_priority": False, "user": { - "initial_batch_size": nworkers - 1, "max_resource_sets": max_rsets, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": {"give_all_with_same_priority": False}, - } - # comms = libE_specs["disable_resource_manager"] = True # SH TCP testing comms = libE_specs["comms"] @@ -95,9 +90,7 @@ exit_criteria = {"sim_max": 40, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs, alloc_specs=alloc_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert flag == 0 diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py b/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py index b4c30f9d2d..793f0e98c8 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.tests.regression_tests.support import write_sim_func as sim_f @@ -54,18 +55,24 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 21} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert os.path.isdir(c_ensemble), f"Ensemble directory {c_ensemble} not created." diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py b/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py index 69bb34ab84..59cbe15214 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.tests.regression_tests.support import write_sim_func as sim_f @@ -53,18 +54,24 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 21} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert os.path.isdir(w_ensemble), f"Ensemble directory {w_ensemble} not created." diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py b/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py index 229f6d5f54..6e772cdba0 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.manager import LoggedException @@ -46,20 +47,26 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 21} return_flag = 1 try: - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs + ) except LoggedException as e: print(f"Caught deliberate exception: {e}") return_flag = 0 diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py b/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py index c73eeabadf..836a907909 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.libE import libE from libensemble.tests.regression_tests.support import write_sim_func as sim_f from libensemble.tests.regression_tests.support import write_uniform_gen_func as gen_f @@ -65,8 +66,8 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, @@ -76,7 +77,13 @@ exit_criteria = {"sim_max": 20} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + alloc_specs = { + "alloc_f": give_sim_work_first, + } + + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) def check_copied(type): input_copied = [] diff --git a/libensemble/tests/functionality_tests/test_sim_input_dir_option.py b/libensemble/tests/functionality_tests/test_sim_input_dir_option.py index 7dc4da7057..023bc98db5 100644 --- a/libensemble/tests/functionality_tests/test_sim_input_dir_option.py +++ b/libensemble/tests/functionality_tests/test_sim_input_dir_option.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.tests.regression_tests.support import write_sim_func as sim_f @@ -49,8 +50,8 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, @@ -60,7 +61,13 @@ exit_criteria = {"sim_max": 21} - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + alloc_specs = { + "alloc_f": give_sim_work_first, + } + + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) if is_manager: assert os.path.isdir(o_ensemble), f"Ensemble directory {o_ensemble} not created." diff --git a/libensemble/tests/functionality_tests/test_stats_output.py b/libensemble/tests/functionality_tests/test_stats_output.py index 58d012cdeb..b6377f8665 100644 --- a/libensemble/tests/functionality_tests/test_stats_output.py +++ b/libensemble/tests/functionality_tests/test_stats_output.py @@ -70,23 +70,19 @@ ("x", float, n), ("x_on_cube", float, n), ], + "batch_mode": False, + "give_all_with_same_priority": True, + "num_active_gens": 1, + "async_return": True, + "batch_size": 5, "user": { - "gen_batch_size": 5, "max_resource_sets": nworkers, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = { - "alloc_f": give_sim_work_first, - "user": { - "batch_mode": False, - "give_all_with_same_priority": True, - "num_active_gens": 1, - "async_return": True, - }, - } + alloc_specs = {"alloc_f": give_sim_work_first} # This can improve scheduling when tasks may run across multiple nodes libE_specs["scheduler_opts"] = {"match_slots": False} diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling.py b/libensemble/tests/functionality_tests/test_uniform_sampling.py index 2867f94df9..ea9ef4bbb2 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling.py @@ -19,6 +19,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample # Import libEnsemble items for this test @@ -47,8 +48,8 @@ gen_specs = { "gen_f": uniform_random_sample, # Function generating sim_f input "out": [("x", float, (2,))], # Tell libE gen_f output, type, size + "batch_size": 500, "user": { - "gen_batch_size": 500, # Used by this specific gen_f "lb": np.array([-3, -2]), # Used by this specific gen_f "ub": np.array([3, 2]), # Used by this specific gen_f }, @@ -59,6 +60,10 @@ exit_criteria = {"gen_max": 501, "wallclock_max": 300} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + for run in range(2): if run == 1: # Test running a mock sim using previous history file @@ -67,7 +72,9 @@ sim_specs["user"] = {"history_file": hfile} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs=libE_specs + ) if is_manager: assert flag == 0 diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py b/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py index 521b6e169d..855abfd44e 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py @@ -76,8 +76,10 @@ def create_H0(persis_info, gen_specs, sim_max): gen_specs = { "gen_f": uniform_random_sample_cancel, # Function generating sim_f input "out": [("x", float, (2,)), ("cancel_requested", bool)], + "batch_size": 50, + "batch_mode": True, + "num_active_gens": 1, "user": { - "gen_batch_size": 50, # Used by this specific gen_f "lb": np.array([-3, -2]), # Used by this specific gen_f "ub": np.array([3, 2]), # Used by this specific gen_f }, @@ -90,16 +92,11 @@ def create_H0(persis_info, gen_specs, sim_max): a_spec_1 = { "alloc_f": gswf, - "user": { - "batch_mode": True, - "num_active_gens": 1, - }, } a_spec_2 = { "alloc_f": gswf, "user": { - "batch_mode": True, "num_active_gens": 2, }, } diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_one_residual_at_a_time.py b/libensemble/tests/functionality_tests/test_uniform_sampling_one_residual_at_a_time.py index 6e96210db3..87093b1fff 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_one_residual_at_a_time.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_one_residual_at_a_time.py @@ -57,8 +57,10 @@ "gen_f": gen_f, "in": ["pt_id"], "out": [("x", float, n), ("priority", float), ("paused", bool), ("obj_component", int), ("pt_id", int)], + "batch_size": 2, + "batch_mode": True, # Wait until all sim evals are done + "num_active_gens": 1, # Only allow one active generator "user": { - "gen_batch_size": 2, "single_component_at_a_time": True, "combine_component_func": lambda x: np.sum(np.power(x, 2)), "lb": (-2 - np.pi / 10) * np.ones(n), @@ -71,8 +73,6 @@ "alloc_f": give_sim_work_first, # Allocation function "user": { "stop_on_NaNs": True, # Should alloc preempt evals - "batch_mode": True, # Wait until all sim evals are done - "num_active_gens": 1, # Only allow one active generator "stop_partial_fvec_eval": True, # Should alloc preempt evals }, } diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py b/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py index ec126607ef..8a5bf14f3d 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py @@ -54,17 +54,19 @@ "gen_f": gen_f, "persis_in": ["x", "f", "grad", "sim_id"], "out": gen_out, + "batch_size": 2, + "batch_mode": True, + "num_active_gens": 1, "user": { "xtol_rel": 1e-4, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), - "gen_batch_size": 2, "localopt_method": "LD_MMA", "xtol_rel": 1e-4, }, } - alloc_specs = {"alloc_f": alloc_f, "user": {"batch_mode": True, "num_active_gens": 1}} + alloc_specs = {"alloc_f": alloc_f} persis_info = add_unique_random_streams({}, nworkers + 1) diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py index 9123f7db17..c895d12a8c 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py @@ -67,8 +67,12 @@ ("x", float, n), ("x_on_cube", float, n), ], + "batch_size": 5, + "batch_mode": False, + "give_all_with_same_priority": True, + "num_active_gens": 1, + "async_return": True, "user": { - "gen_batch_size": 5, "max_resource_sets": nworkers, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), @@ -77,12 +81,6 @@ alloc_specs = { "alloc_f": give_sim_work_first, - "user": { - "batch_mode": False, - "give_all_with_same_priority": True, - "num_active_gens": 1, - "async_return": True, - }, } # This can improve scheduling when tasks may run across multiple nodes diff --git a/libensemble/tests/functionality_tests/test_worker_exceptions.py b/libensemble/tests/functionality_tests/test_worker_exceptions.py index 435e621e16..88cb6fcb0b 100644 --- a/libensemble/tests/functionality_tests/test_worker_exceptions.py +++ b/libensemble/tests/functionality_tests/test_worker_exceptions.py @@ -16,6 +16,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.manager import LoggedException @@ -52,10 +53,16 @@ # Tell libEnsemble when to stop exit_criteria = {"wallclock_max": 10} + alloc_specs = { + "alloc_f": give_sim_work_first, + } + # Perform the run return_flag = 1 try: - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) except LoggedException as e: print(f"Caught deliberate exception: {e}") return_flag = 0 diff --git a/libensemble/tests/functionality_tests/test_workflow_dir.py b/libensemble/tests/functionality_tests/test_workflow_dir.py index 1abbc10bcf..00c6a5e854 100644 --- a/libensemble/tests/functionality_tests/test_workflow_dir.py +++ b/libensemble/tests/functionality_tests/test_workflow_dir.py @@ -18,6 +18,7 @@ import numpy as np +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f from libensemble.libE import libE from libensemble.tests.regression_tests.support import write_sim_func as sim_f @@ -51,13 +52,17 @@ gen_specs = { "gen_f": gen_f, "out": [("x", float, (1,))], + "batch_size": 20, "user": { - "gen_batch_size": 20, "lb": np.array([-3]), "ub": np.array([3]), }, } + alloc_specs = { + "alloc_f": give_sim_work_first, + } + persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": 21} @@ -70,7 +75,9 @@ "./test_workflow" + str(i) + "_nworkers" + str(nworkers) + "_comms-" + libE_specs["comms"] ) - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) + H, persis_info, flag = libE( + sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs + ) assert os.path.isdir(libE_specs["workflow_dir_path"]), "workflow_dir not created" assert all( diff --git a/libensemble/tests/functionality_tests/test_zero_resource_workers.py b/libensemble/tests/functionality_tests/test_zero_resource_workers.py index c8f0786d06..a1b3e8e7cd 100644 --- a/libensemble/tests/functionality_tests/test_zero_resource_workers.py +++ b/libensemble/tests/functionality_tests/test_zero_resource_workers.py @@ -14,7 +14,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f from libensemble.libE import libE @@ -93,14 +92,13 @@ "gen_f": gen_f, "in": [], "out": [("x", float, (n,))], + "initial_batch_size": 20, "user": { - "initial_batch_size": 20, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = {"alloc_f": alloc_f} persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": (nsim_workers) * rounds} @@ -125,6 +123,6 @@ } # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py b/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py index 69ea2b559c..978a992d01 100644 --- a/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py +++ b/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py @@ -16,7 +16,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f from libensemble.libE import libE @@ -92,14 +91,13 @@ "gen_f": gen_f, "in": [], "out": [("x", float, (n,))], + "initial_batch_size": 20, "user": { - "initial_batch_size": 20, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, } - alloc_specs = {"alloc_f": alloc_f} persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": (nsim_workers) * rounds} @@ -128,6 +126,6 @@ } # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # All asserts are in sim func diff --git a/libensemble/tests/regression_tests/support.py b/libensemble/tests/regression_tests/support.py index 4189bcfe48..f472e51290 100644 --- a/libensemble/tests/regression_tests/support.py +++ b/libensemble/tests/regression_tests/support.py @@ -69,7 +69,7 @@ def write_uniform_gen_func(H, persis_info, gen_specs, _): ub = gen_specs["user"]["ub"] lb = gen_specs["user"]["lb"] n = len(lb) - b = gen_specs["user"]["gen_batch_size"] + b = gen_specs["batch_size"] H_o = np.zeros(b, dtype=gen_specs["out"]) H_o["x"] = persis_info["rand_stream"].uniform(lb, ub, (b, n)) with open("test_gen_out.txt", "a") as f: @@ -105,7 +105,7 @@ def write_uniform_gen_func(H, persis_info, gen_specs, _): "next_to_give": 0, # Remembers next H row to give in alloc_f } -persis_info_1[0] = { +persis_info_1[0] = { # noqa "run_order": {}, # Used by manager to remember run order "total_runs": 0, # Used by manager to count total runs "rand_stream": np.random.default_rng(1), diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index edecabb668..a9abbb055b 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -11,12 +11,12 @@ # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local threads tcp -# TESTSUITE_NPROCS: 2 4 +# TESTSUITE_NPROCS: 3 4 import numpy as np from libensemble import Ensemble -from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f +from libensemble.gen_funcs.persistent_sampling import persistent_uniform # Import libEnsemble items for this test from libensemble.sim_funcs.simple_sim import norm_eval as sim_f @@ -29,10 +29,10 @@ sampling.libE_specs = LibeSpecs(save_every_k_gens=300, safe_mode=False, disable_log_files=True) sampling.sim_specs = SimSpecs(sim_f=sim_f) sampling.gen_specs = GenSpecs( - gen_f=gen_f, + gen_f=persistent_uniform, outputs=[("x", float, (1,))], + initial_batch_size=100, user={ - "gen_batch_size": 100, "lb": np.array([-3]), "ub": np.array([3]), }, diff --git a/libensemble/tests/regression_tests/test_2d_sampling.py b/libensemble/tests/regression_tests/test_2d_sampling.py index 8164c2844a..3601528e10 100644 --- a/libensemble/tests/regression_tests/test_2d_sampling.py +++ b/libensemble/tests/regression_tests/test_2d_sampling.py @@ -16,11 +16,12 @@ import numpy as np from libensemble import Ensemble +from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.gen_funcs.sampling import latin_hypercube_sample as gen_f # Import libEnsemble items for this test from libensemble.sim_funcs.simple_sim import norm_eval as sim_f -from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs # Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). if __name__ == "__main__": @@ -30,13 +31,15 @@ sampling.gen_specs = GenSpecs( gen_f=gen_f, outputs=[("x", float, 2)], + batch_size=100, user={ - "gen_batch_size": 100, "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, ) + sampling.alloc_specs = AllocSpecs(alloc_f=give_sim_work_first) + sampling.exit_criteria = ExitCriteria(sim_max=200) sampling.add_random_streams() diff --git a/libensemble/tests/regression_tests/test_GPU_variable_resources.py b/libensemble/tests/regression_tests/test_GPU_variable_resources.py index b6b3197f90..940688501e 100644 --- a/libensemble/tests/regression_tests/test_GPU_variable_resources.py +++ b/libensemble/tests/regression_tests/test_GPU_variable_resources.py @@ -28,7 +28,6 @@ import numpy as np from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_with_procs_gpus as gen_f1 from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample_with_var_gpus as gen_f2 @@ -36,7 +35,7 @@ # Import libEnsemble items for this test from libensemble.sim_funcs import six_hump_camel from libensemble.sim_funcs.var_resources import gpu_variable_resources_from_gen as sim_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs from libensemble.tools import add_unique_random_streams # from libensemble import logger @@ -69,22 +68,16 @@ gen_f=gen_f1, persis_in=["f", "x", "sim_id"], out=[("num_procs", int), ("num_gpus", int), ("x", float, 2)], + initial_batch_size=gpu_test.nworkers - 1, + give_all_with_same_priority=False, + async_return=False, user={ - "initial_batch_size": gpu_test.nworkers - 1, "max_procs": gpu_test.nworkers - 1, # Any sim created can req. 1 worker up to max "lb": np.array([-3, -2]), "ub": np.array([3, 2]), }, ) - gpu_test.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "give_all_with_same_priority": False, - "async_return": False, # False causes batch returns - }, - ) - # Run with random num_procs/num_gpus for each simulation gpu_test.persis_info = add_unique_random_streams({}, gpu_test.nworkers + 1) gpu_test.exit_criteria = ExitCriteria(sim_max=20) diff --git a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py index 2b583d4f06..87b450db58 100644 --- a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py +++ b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py @@ -37,7 +37,6 @@ import numpy as np from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor # Using num_procs / num_gpus in gen @@ -46,7 +45,7 @@ # Import libEnsemble items for this test from libensemble.sim_funcs import six_hump_camel from libensemble.sim_funcs.var_resources import gpu_variable_resources_from_gen as sim_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs from libensemble.tools import add_unique_random_streams # from libensemble import logger @@ -79,8 +78,10 @@ gen_f=gen_f, persis_in=["f", "x", "sim_id"], out=[("num_procs", int), ("num_gpus", int), ("x", float, 2)], + initial_batch_size=nworkers - 1, + give_all_with_same_priority=False, + async_return=False, user={ - "initial_batch_size": nworkers - 1, "max_procs": (nworkers - 1) // 2, # Any sim created can req. 1 worker up to max "lb": np.array([-3, -2]), "ub": np.array([3, 2]), @@ -88,14 +89,6 @@ }, ) - gpu_test.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "give_all_with_same_priority": False, - "async_return": False, # False causes batch returns - }, - ) - gpu_test.persis_info = add_unique_random_streams({}, gpu_test.nworkers + 1) gpu_test.exit_criteria = ExitCriteria(sim_max=40, wallclock_max=300) diff --git a/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py b/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py index a85771d7dd..ec7615fdb7 100644 --- a/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py +++ b/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py @@ -31,10 +31,9 @@ from gest_api.vocs import VOCS from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_classes import APOSMM from libensemble.manager import LoggedException -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, SimSpecs from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima @@ -62,7 +61,6 @@ def six_hump_camel_func(x): start_time = time() n = 2 - workflow.alloc_specs = AllocSpecs(alloc_f=alloc_f) workflow.libE_specs.gen_on_manager = True diff --git a/libensemble/tests/regression_tests/test_asktell_gpCAM.py b/libensemble/tests/regression_tests/test_asktell_gpCAM.py index b093a0df7a..f59fc135d7 100644 --- a/libensemble/tests/regression_tests/test_asktell_gpCAM.py +++ b/libensemble/tests/regression_tests/test_asktell_gpCAM.py @@ -24,7 +24,6 @@ import numpy as np from gest_api.vocs import VOCS -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_classes.gpCAM import GP_CAM, GP_CAM_Covar # Import libEnsemble items for this test @@ -65,8 +64,6 @@ vocs = VOCS(variables={"x0": [-3, 3], "x1": [-2, 2], "x2": [-1, 1], "x3": [-1, 1]}, objectives={"f": "MINIMIZE"}) - alloc_specs = {"alloc_f": alloc_f} - gen = GP_CAM_Covar(vocs) for inst in range(3): @@ -89,7 +86,7 @@ exit_criteria = {"sim_max": num_batches * batch_size, "wallclock_max": 300} # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, {}, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, {}, libE_specs=libE_specs) if is_manager: assert len(np.unique(H["gen_ended_time"])) == num_batches diff --git a/libensemble/tests/regression_tests/test_ensemble_platform_workdir.py b/libensemble/tests/regression_tests/test_ensemble_platform_workdir.py index 121b0cd37a..bb5a63488a 100644 --- a/libensemble/tests/regression_tests/test_ensemble_platform_workdir.py +++ b/libensemble/tests/regression_tests/test_ensemble_platform_workdir.py @@ -6,7 +6,6 @@ # Import libEnsemble items for this test from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.mpi_executor import MPIExecutor from libensemble.gen_funcs.persistent_sampling_var_resources import uniform_sample as gen_f from libensemble.resources.platforms import PerlmutterGPU @@ -23,14 +22,6 @@ six_hump_camel_app = six_hump_camel.__file__ n = 2 - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "give_all_with_same_priority": False, - "async_return": False, # False batch returns - }, - } - exit_criteria = {"sim_max": 20} # Ensure LIBE_PLATFORM environment variable is not set. @@ -43,9 +34,7 @@ ensemble = Ensemble( parse_args=True, executor=exctr, - alloc_specs=alloc_specs, exit_criteria=exit_criteria, - # libE_specs = LibeSpecs(use_workflow_dir=True, platform_specs=platform_specs), # works ) platform_specs = PerlmutterGPU() @@ -60,8 +49,10 @@ "gen_f": gen_f, "persis_in": ["f", "x", "sim_id"], "out": [("priority", float), ("resource_sets", int), ("x", float, n)], + "initial_batch_size": ensemble.nworkers - 1, + "give_all_with_same_priority": False, + "async_return": False, "user": { - "initial_batch_size": ensemble.nworkers - 1, "max_resource_sets": ensemble.nworkers - 1, # Any sim created can req. 1 worker up to all. "lb": np.array([-3, -2]), "ub": np.array([3, 2]), diff --git a/libensemble/tests/regression_tests/test_gpCAM.py b/libensemble/tests/regression_tests/test_gpCAM.py index 218ecfc918..b5074ac610 100644 --- a/libensemble/tests/regression_tests/test_gpCAM.py +++ b/libensemble/tests/regression_tests/test_gpCAM.py @@ -28,7 +28,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM, persistent_gpCAM_covar # Import libEnsemble items for this test @@ -67,8 +66,6 @@ }, } - alloc_specs = {"alloc_f": alloc_f} - for inst in range(3): if inst == 0: gen_specs["gen_f"] = persistent_gpCAM_covar @@ -91,7 +88,7 @@ persis_info = add_unique_random_streams({}, nworkers + 1) # Perform the run - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: assert len(np.unique(H["gen_ended_time"])) == num_batches diff --git a/libensemble/tests/regression_tests/test_optimas_ax_mf.py b/libensemble/tests/regression_tests/test_optimas_ax_mf.py index 758aa1fc2c..5b9f4dca08 100644 --- a/libensemble/tests/regression_tests/test_optimas_ax_mf.py +++ b/libensemble/tests/regression_tests/test_optimas_ax_mf.py @@ -23,8 +23,7 @@ from optimas.generators import AxMultiFidelityGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs def eval_func_mf(input_params): @@ -62,13 +61,11 @@ def eval_func_mf(input_params): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=6) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/regression_tests/test_optimas_ax_multitask.py b/libensemble/tests/regression_tests/test_optimas_ax_multitask.py index 43c6c444eb..e7031bb226 100644 --- a/libensemble/tests/regression_tests/test_optimas_ax_multitask.py +++ b/libensemble/tests/regression_tests/test_optimas_ax_multitask.py @@ -31,8 +31,7 @@ from optimas.generators import AxMultitaskGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs def eval_func_multitask(input_params): @@ -73,7 +72,6 @@ def eval_func_multitask(input_params): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=15) H0 = None # or np.load("multitask_first_pass.npy") @@ -92,7 +90,6 @@ def eval_func_multitask(input_params): workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, H0=H0, diff --git a/libensemble/tests/regression_tests/test_optimas_ax_sf.py b/libensemble/tests/regression_tests/test_optimas_ax_sf.py index e4ee9e8a79..1a23edf290 100644 --- a/libensemble/tests/regression_tests/test_optimas_ax_sf.py +++ b/libensemble/tests/regression_tests/test_optimas_ax_sf.py @@ -23,8 +23,7 @@ from optimas.generators import AxSingleFidelityGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs def eval_func_sf(input_params): @@ -64,13 +63,11 @@ def eval_func_sf(input_params): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=10) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/regression_tests/test_optimas_grid_sample.py b/libensemble/tests/regression_tests/test_optimas_grid_sample.py index 57c6c8fedf..c390be8ca1 100644 --- a/libensemble/tests/regression_tests/test_optimas_grid_sample.py +++ b/libensemble/tests/regression_tests/test_optimas_grid_sample.py @@ -24,8 +24,7 @@ from optimas.generators import GridSamplingGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs def eval_func(input_params: dict): @@ -73,13 +72,11 @@ def eval_func(input_params: dict): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=n_evals) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py index 6e19930691..c810db561e 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py @@ -69,6 +69,7 @@ def combine_component(x): "gen_f": gen_f, "persis_in": ["f", "fvec"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "localopt_method": "dfols", diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py index b197dc3f07..09544e51d1 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py @@ -69,6 +69,7 @@ def assertion(passed): "gen_f": gen_f, "persis_in": ["f"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "localopt_method": "LN_BOBYQA", diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_external_localopt.py b/libensemble/tests/regression_tests/test_persistent_aposmm_external_localopt.py index dd01d1069e..22cf47325f 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_external_localopt.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_external_localopt.py @@ -75,6 +75,7 @@ "gen_f": gen_f, "persis_in": ["f"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "sample_points": np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py b/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py index 9d42784439..bf5914573f 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py @@ -64,6 +64,7 @@ "gen_f": gen_f, "persis_in": ["f"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "sample_points": np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py index 5b038a0ce8..3b7609934d 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py @@ -92,9 +92,11 @@ def combine_component(x): "lb": lb, "ub": ub, }, + "batch_mode": True, + "num_active_gens": 1, } - alloc_specs = {"alloc_f": alloc_f, "user": {"batch_mode": True, "num_active_gens": 1}} + alloc_specs = {"alloc_f": alloc_f} persis_info = add_unique_random_streams({}, nworkers + 1) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py index 76107f567d..f40da2c4b6 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py @@ -65,6 +65,7 @@ "gen_f": gen_f, "persis_in": ["f"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "sample_points": np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py index 39ff3b79fd..8a3c762871 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py @@ -67,6 +67,7 @@ "gen_f": gen_f, "persis_in": ["f", "grad"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "sample_points": np.round(minima, 1), diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py index d6db6b63a1..0dcb4a2c0d 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py @@ -60,6 +60,7 @@ "gen_f": gen_f, "persis_in": ["f", "grad"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "localopt_method": "nm", diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py index e61843fd71..a31dd01878 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py @@ -62,6 +62,7 @@ "gen_f": gen_f, "persis_in": ["f"] + [n[0] for n in gen_out], "out": gen_out, + "initial_batch_size": 100, "user": { "initial_sample_size": 100, "localopt_method": "LN_BOBYQA", diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index f2d2f09cc0..a0364e1c14 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -72,6 +72,7 @@ "in": gen_in, "persis_in": gen_in, "out": gen_out, + "initial_batch_size": 0, "user": { "initial_sample_size": 0, # Don't need to do evaluations because the sampling already done below "localopt_method": "LD_MMA", diff --git a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py index 990493a176..38b6140fcd 100644 --- a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py +++ b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py @@ -26,7 +26,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens from libensemble.libE import libE from libensemble.message_numbers import WORKER_DONE from libensemble.tools import add_unique_random_streams, parse_args, save_libE_output @@ -90,13 +89,13 @@ def run_simulation(H, persis_info, sim_specs, libE_info): "out": [ # parameters to input into the simulation. ("x", float, (2,)), - ("task", str, max([len(mt_params["name_hifi"]), len(mt_params["name_lofi"])])), + ("task", str, max(len(str(mt_params["name_hifi"])), len(str(mt_params["name_lofi"])))), ("resource_sets", int), ], + "async_return": False, + "batch_size": nworkers - 1, "user": { "range": [1, 8], - # Total max number of sims running concurrently. - "gen_batch_size": nworkers - 1, # Lower bound for the n parameters. "lb": np.array([0, 0]), # Upper bound for the n parameters. @@ -105,11 +104,6 @@ def run_simulation(H, persis_info, sim_specs, libE_info): } gen_specs["user"] = {**gen_specs["user"], **mt_params} - alloc_specs = { - "alloc_f": only_persistent_gens, - "user": {"async_return": False}, - } - # libE logger logger.set_level("INFO") @@ -120,7 +114,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): persis_info = add_unique_random_streams({}, nworkers + 1) # Run LibEnsemble, and store results in history array H - H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # Save results to numpy file if is_manager: diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py index 39cf11b5de..6936269466 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -33,13 +33,12 @@ import numpy as np from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib as gen_f # Import libEnsemble items for this test from libensemble.sim_funcs.surmise_test_function import borehole as sim_f from libensemble.sim_funcs.surmise_test_function import tstd2theta -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, SimSpecs from libensemble.tools import add_unique_random_streams if __name__ == "__main__": @@ -84,18 +83,12 @@ "step_add_theta": step_add_theta, # No. of thetas to generate per step "n_explore_theta": n_explore_theta, # No. of thetas to explore each step "obsvar": obsvar, # Variance for generating noise in obs - "init_sample_size": init_sample_size, # Initial batch size inc. observations "priorloc": 1, # Prior location in the unit cube "priorscale": 0.5, # Standard deviation of prior }, - ), - alloc_specs=AllocSpecs( - alloc_f=alloc_f, - user={ - "init_sample_size": init_sample_size, - "async_return": True, # True = Return results to gen as they come in (after sample) - "active_recv_gen": True, # Persistent gen can handle irregular communications - }, + initial_batch_size=init_sample_size, + async_return=True, + active_recv_gen=True, ), exit_criteria=ExitCriteria(sim_max=max_evals), ) diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py index 11095f61f2..6e289a0081 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -34,7 +34,6 @@ import numpy as np -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors.executor import Executor from libensemble.gen_funcs.persistent_surmise_calib import surmise_calib as gen_f @@ -105,34 +104,25 @@ "gen_f": gen_f, "persis_in": [o[0] for o in gen_out] + ["f", "sim_ended", "sim_id"], "out": gen_out, + "initial_batch_size": init_sample_size, + "async_return": True, + "active_recv_gen": True, "user": { "n_init_thetas": n_init_thetas, # Num thetas in initial batch "num_x_vals": n_x, # Num x points to create "step_add_theta": step_add_theta, # No. of thetas to generate per step "n_explore_theta": n_explore_theta, # No. of thetas to explore each step "obsvar": obsvar, # Variance for generating noise in obs - "init_sample_size": init_sample_size, # Initial batch size inc. observations "priorloc": 1, # Prior location in the unit cube. "priorscale": 0.2, # Standard deviation of prior }, } - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "init_sample_size": init_sample_size, - "async_return": True, # True = Return results to gen as they come in (after sample) - "active_recv_gen": True, # Persistent gen can handle irregular communications - }, - } - persis_info = add_unique_random_streams({}, nworkers + 1) exit_criteria = {"sim_max": max_evals} # Perform the run - H, persis_info, flag = libE( - sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs=alloc_specs, libE_specs=libE_specs - ) + H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) if is_manager: print("Cancelled sims", H["sim_id"][H["cancel_requested"]]) diff --git a/libensemble/tests/regression_tests/test_xopt_EI.py b/libensemble/tests/regression_tests/test_xopt_EI.py index a78aee60a5..69eea46dc1 100644 --- a/libensemble/tests/regression_tests/test_xopt_EI.py +++ b/libensemble/tests/regression_tests/test_xopt_EI.py @@ -23,8 +23,7 @@ from xopt.generators.bayesian.expected_improvement import ExpectedImprovementGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs # Adapted from Xopt/xopt/resources/testing.py @@ -85,13 +84,11 @@ def xtest_sim(H, persis_info, sim_specs, _): vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=20) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/regression_tests/test_xopt_EI_xopt_sim.py b/libensemble/tests/regression_tests/test_xopt_EI_xopt_sim.py index 07ace47fa9..9efffb58e0 100644 --- a/libensemble/tests/regression_tests/test_xopt_EI_xopt_sim.py +++ b/libensemble/tests/regression_tests/test_xopt_EI_xopt_sim.py @@ -23,8 +23,7 @@ from xopt.generators.bayesian.expected_improvement import ExpectedImprovementGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs # From Xopt/xopt/resources/testing.py @@ -79,13 +78,11 @@ def xtest_callable(input_dict: dict, a=0) -> dict: vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=20) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/regression_tests/test_xopt_nelder_mead.py b/libensemble/tests/regression_tests/test_xopt_nelder_mead.py index fe8039b95c..4f0a4d285a 100644 --- a/libensemble/tests/regression_tests/test_xopt_nelder_mead.py +++ b/libensemble/tests/regression_tests/test_xopt_nelder_mead.py @@ -21,8 +21,7 @@ from xopt.generators.sequential.neldermead import NelderMeadGenerator from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs def rosenbrock_callable(input_dict: dict) -> dict: @@ -67,13 +66,11 @@ def rosenbrock_callable(input_dict: dict) -> dict: vocs=vocs, ) - alloc_specs = AllocSpecs(alloc_f=alloc_f) exit_criteria = ExitCriteria(sim_max=30) workflow = Ensemble( libE_specs=libE_specs, sim_specs=sim_specs, - alloc_specs=alloc_specs, gen_specs=gen_specs, exit_criteria=exit_criteria, ) diff --git a/libensemble/tests/scaling_tests/forces/forces_adv/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/forces_adv/run_libe_forces.py index 3b5a489d62..7be531c37b 100644 --- a/libensemble/tests/scaling_tests/forces/forces_adv/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/forces_adv/run_libe_forces.py @@ -20,8 +20,8 @@ from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f else: - from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first as alloc_f - from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f + from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first as alloc_f # type: ignore[no-redef] + from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f # type: ignore[no-redef] logger.set_level("INFO") # INFO is now default @@ -65,23 +65,20 @@ "gen_f": gen_f, # Generator function "in": [], # Generator input "out": [("x", float, (1,))], # Name, type and size of data produced (must match sim_specs 'in') + "batch_size": 1000, # How many random samples to generate in one call "user": { "lb": np.array([0]), # Lower bound for random sample array (1D) "ub": np.array([32767]), # Upper bound for random sample array (1D) - "gen_batch_size": 1000, # How many random samples to generate in one call }, } if PERSIS_GEN: - alloc_specs = {"alloc_f": alloc_f} + alloc_specs = {} + gen_specs["async_return"] = False else: - alloc_specs = { - "alloc_f": alloc_f, - "user": { - "batch_mode": True, # If true wait for all sims to process before generate more - "num_active_gens": 1, # Only one active generator at a time - }, - } + alloc_specs = {"alloc_f": alloc_f} + gen_specs["batch_mode"] = True + gen_specs["num_active_gens"] = 1 libE_specs["save_every_k_gens"] = 1000 # Save every K steps libE_specs["sim_dirs_make"] = True # Separate each sim into a separate directory @@ -92,8 +89,7 @@ exit_criteria = {"sim_max": sim_max} # Create a different random number stream for each worker and the manager -persis_info = {} -persis_info = add_unique_random_streams(persis_info, nworkers + 1) +persis_info = add_unique_random_streams({}, nworkers + 1) try: H, persis_info, flag = libE( diff --git a/libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py index a70477748b..6d5e0bb0e5 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py @@ -6,10 +6,9 @@ from forces_simf import run_forces # Sim func from current dir from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs if __name__ == "__main__": # Initialize MPI Executor @@ -44,20 +43,15 @@ inputs=[], # No input when start persistent generator persis_in=["sim_id"], # Return sim_ids of evaluated points to generator outputs=[("x", float, (1,))], + initial_batch_size=nsim_workers, + async_return=False, user={ - "initial_batch_size": nsim_workers, "lb": np.array([1000]), # min particles "ub": np.array([3000]), # max particles }, ) # Starts one persistent generator. Simulated values are returned in batch. - ensemble.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "async_return": False, # False causes batch returns - }, - ) # Instruct libEnsemble to exit after this many simulations ensemble.exit_criteria = ExitCriteria(sim_max=8) diff --git a/libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/run_libe_forces.py index 8f3e8d442a..41217173b8 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/run_libe_forces.py @@ -6,10 +6,9 @@ from forces_simf import run_forces # Sim func from current dir from libensemble import Ensemble -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.executors import MPIExecutor from libensemble.gen_funcs.persistent_sampling import persistent_uniform as gen_f -from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs +from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs if __name__ == "__main__": # Initialize MPI Executor @@ -48,20 +47,15 @@ inputs=[], # No input when start persistent generator persis_in=["sim_id"], # Return sim_ids of evaluated points to generator outputs=[("x", float, (1,))], + initial_batch_size=nsim_workers, + async_return=False, user={ - "initial_batch_size": nsim_workers, "lb": np.array([1000]), # min particles "ub": np.array([3000]), # max particles }, ) # Starts one persistent generator. Simulated values are returned in batch. - ensemble.alloc_specs = AllocSpecs( - alloc_f=alloc_f, - user={ - "async_return": False, # False causes batch returns - }, - ) # Instruct libEnsemble to exit after this many simulations ensemble.exit_criteria = ExitCriteria(sim_max=8) diff --git a/libensemble/tests/scaling_tests/persistent_gp/run_example.py b/libensemble/tests/scaling_tests/persistent_gp/run_example.py index 035fa9c1e0..182c0034e1 100644 --- a/libensemble/tests/scaling_tests/persistent_gp/run_example.py +++ b/libensemble/tests/scaling_tests/persistent_gp/run_example.py @@ -9,7 +9,6 @@ import numpy as np from libensemble import logger -from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens from libensemble.gen_funcs.persistent_ax_multitask import persistent_gp_mt_ax_gen_f from libensemble.libE import libE from libensemble.message_numbers import WORKER_DONE @@ -63,13 +62,13 @@ def run_simulation(H, persis_info, sim_specs, libE_info): "out": [ # parameters to input into the simulation. ("x", float, (2,)), - ("task", str, max([len(mt_params["name_hifi"]), len(mt_params["name_lofi"])])), + ("task", str, max(len(str(mt_params["name_hifi"])), len(str(mt_params["name_lofi"])))), ("resource_sets", int), ], + "async_return": False, + "batch_size": nworkers - 1, "user": { "range": [1, 8], - # Total max number of sims running concurrently. - "gen_batch_size": nworkers - 1, # Lower bound for the n parameters. "lb": np.array([0, 0]), # Upper bound for the n parameters. @@ -78,11 +77,6 @@ def run_simulation(H, persis_info, sim_specs, libE_info): } gen_specs["user"] = {**gen_specs["user"], **mt_params} -alloc_specs = { - "alloc_f": only_persistent_gens, - "out": [("gen_informed", bool)], - "user": {"async_return": False}, -} # libE logger logger.set_level("INFO") @@ -94,7 +88,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): persis_info = add_unique_random_streams({}, nworkers + 1) # Run LibEnsemble, and store results in history array H -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, alloc_specs, libE_specs) +H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, libE_specs=libE_specs) # Save results to numpy file if is_manager: diff --git a/libensemble/tests/unit_tests/test_ensemble.py b/libensemble/tests/unit_tests/test_ensemble.py index 94c4865ad4..3a64a38c18 100644 --- a/libensemble/tests/unit_tests/test_ensemble.py +++ b/libensemble/tests/unit_tests/test_ensemble.py @@ -38,10 +38,11 @@ def test_ensemble_parse_args_false(): def test_full_workflow(): """Test initializing a workflow via Specs and Ensemble.run()""" + from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.ensemble import Ensemble from libensemble.gen_funcs.sampling import latin_hypercube_sample from libensemble.sim_funcs.simple_sim import norm_eval - from libensemble.specs import ExitCriteria, GenSpecs, LibeSpecs, SimSpecs + from libensemble.specs import AllocSpecs, ExitCriteria, GenSpecs, LibeSpecs, SimSpecs LS = LibeSpecs(comms="local", nworkers=4) @@ -51,13 +52,14 @@ def test_full_workflow(): sim_specs=SimSpecs(sim_f=norm_eval), gen_specs=GenSpecs( gen_f=latin_hypercube_sample, + batch_size=100, user={ - "gen_batch_size": 100, "lb": np.array([-3]), "ub": np.array([3]), }, ), exit_criteria=ExitCriteria(gen_max=101), + alloc_specs=AllocSpecs(alloc_f=give_sim_work_first), ) ens.add_random_streams() @@ -93,8 +95,8 @@ def test_flakey_workflow(): sim_specs=SimSpecs(sim_f=norm_eval), gen_specs=GenSpecs( gen_f=latin_hypercube_sample, + batch_size=100, user={ - "gen_batch_size": 100, "lb": np.array([-3]), "ub": np.array([3]), }, diff --git a/pixi.lock b/pixi.lock index 185e31d3a4..3357902824 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:817f427252b67646c9eea0f0a4c5a1ce662c2859400dc5fd563f314e92914891 -size 1021669 +oid sha256:61c5432ee07721317765d0bd57cc8802f96ec9376005b4bedc1bb26f39dc116f +size 1020189 diff --git a/pyproject.toml b/pyproject.toml index 5538457e56..871e651525 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,12 +8,7 @@ authors = [ { name = "John-Luke Navarro" }, ] -dependencies = [ - "numpy", - "psutil", - "pydantic", - "gest-api>=0.1,<0.2", -] +dependencies = ["numpy", "psutil", "pydantic", "gest-api>=0.1,<0.2"] description = "A Python toolkit for coordinating asynchronous and dynamic ensembles of calculations." name = "libensemble" @@ -187,6 +182,7 @@ pip = ">=25.2,<26" setuptools = ">=80.8.0,<81" numpy = ">=2.2.6,<3" pydantic = ">=2.12.4,<3" +gest-api = ">=0.1,<0.2" # macOS dependencies @@ -247,9 +243,7 @@ extend-exclude = ["*.bib", "*.xml", "docs/nitpicky"] # Initial, permissive mypy configuration for libensemble. # Allows incremental adoption. To be tightened in future releases. packages = ["libensemble.utils"] -exclude = ''' -libensemble/utils/(launcher|loc_stack|runners|pydantic|output_directory)\.py -''' +exclude = 'libensemble/utils/(launcher|loc_stack|runners|pydantic|output_directory)\.py$|libensemble/tests/regression_tests/support\.py$|libensemble/tests/functionality_tests/.*' disable_error_code = ["import-not-found", "import-untyped"] ignore_missing_imports = true follow_imports = "skip"