From 477de581ec1f1a170b23e5dd84c6f27609461d0d Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 14 Oct 2025 09:17:03 +0200 Subject: [PATCH 1/3] typo --- bfabric/src/bfabric/experimental/workunit_definition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bfabric/src/bfabric/experimental/workunit_definition.py b/bfabric/src/bfabric/experimental/workunit_definition.py index ff65be2cb..51bd61bc3 100644 --- a/bfabric/src/bfabric/experimental/workunit_definition.py +++ b/bfabric/src/bfabric/experimental/workunit_definition.py @@ -26,7 +26,7 @@ class WorkunitExecutionDefinition(BaseModel): """Input dataset (for dataset-flow applications)""" resources: list[int] = [] - """Input resources (for resource-flow applications""" + """Input resources (for resource-flow applications)""" @model_validator(mode="after") def either_dataset_or_resources(self) -> WorkunitExecutionDefinition: From 2f18d45e5f430d0357dffa484bf1fc40b1d34050 Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 14 Oct 2025 09:31:10 +0200 Subject: [PATCH 2/3] basic localworkunit definition --- .../prepare/local_workunit_definition.py | 30 +++++++++++++++++++ .../bfabric_app_runner/specs/inputs_spec.py | 13 ++++---- 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py diff --git a/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py b/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py new file mode 100644 index 000000000..6e18b79da --- /dev/null +++ b/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py @@ -0,0 +1,30 @@ +"""TODO this module is experimental and might be better integrated into different modules later.""" + +from __future__ import annotations + +from pydantic import BaseModel, model_validator + +from bfabric_app_runner.specs.inputs_spec import LocalInputSpecType # noqa: TC001 + + +class LocalWorkunitExecutionDefinition(BaseModel): + """Experimental definition of a local workunit for local execution. + + This is trying to match the structure of WorkunitExecutionRegistration, but, it's notably not compatible right + now due to the WorkunitExecutionDefinition using IDs instead of full objects (and even less so InputSpecs). + """ + + raw_parameters: dict[str, str | None] + """The parameters passed to the workunit, in their raw form, i.e. everything is a string or None.""" + + dataset: LocalInputSpecType | None = None + """Input dataset (for dataset-flow applications)""" + + resources: list[LocalInputSpecType] = [] + """Input resources (for resource-flow applications)""" + + @model_validator(mode="after") + def _mutually_exclusive(self) -> LocalWorkunitExecutionDefinition: + if (self.dataset is None) == (not self.resources): + raise ValueError("either dataset or resources must be provided, but not both") + return self diff --git a/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py index 2bf73bfa4..ea9406792 100644 --- a/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py +++ b/bfabric_app_runner/src/bfabric_app_runner/specs/inputs_spec.py @@ -17,16 +17,19 @@ if TYPE_CHECKING: from pathlib import Path +LocalInputSpecType = Annotated[ + FileSpec | StaticYamlSpec | StaticFileSpec, + Field(discriminator="type"), +] + InputSpecType = Annotated[ - BfabricResourceSpec - | FileSpec + LocalInputSpecType + | BfabricResourceSpec | BfabricDatasetSpec | BfabricOrderFastaSpec | BfabricAnnotationSpec | BfabricResourceArchiveSpec - | BfabricResourceDatasetSpec - | StaticYamlSpec - | StaticFileSpec, + | BfabricResourceDatasetSpec, Field(discriminator="type"), ] From d4cd196326ff9e18480a0e3ecab2d54f44ee6c4b Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Tue, 14 Oct 2025 09:59:00 +0200 Subject: [PATCH 3/3] some initial code --- bfabric_app_runner/pyproject.toml | 4 +- .../src/bfabric_app_runner/cli/__main__.py | 3 +- .../src/bfabric_app_runner/cli/cmd_prepare.py | 63 ++++++++++++++++--- .../prepare/local_workunit_definition.py | 11 ++++ 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/bfabric_app_runner/pyproject.toml b/bfabric_app_runner/pyproject.toml index 34cc02e02..0ce34365a 100644 --- a/bfabric_app_runner/pyproject.toml +++ b/bfabric_app_runner/pyproject.toml @@ -58,12 +58,12 @@ reinstall-package = ["bfabric", "bfabric_scripts", "bfabric_app_runner"] [tool.black] line-length = 120 -target-version = ["py311"] +target-version = ["py312"] [tool.ruff] line-length = 120 indent-width = 4 -target-version = "py311" +target-version = "py312" extend-exclude = [ "examples/template" diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py index 5651ff89e..b93a94421 100644 --- a/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/__main__.py @@ -30,7 +30,7 @@ cmd_action_run_all, cmd_action_dispatch, ) -from bfabric_app_runner.cli.cmd_prepare import cmd_prepare_workunit +from bfabric_app_runner.cli.cmd_prepare import cmd_prepare_workunit, cmd_prepare_local_workunit from bfabric_app_runner.cli.cmd_run import cmd_run_workunit from bfabric_app_runner.cli.inputs import cmd_inputs_prepare, cmd_inputs_clean, cmd_inputs_list, cmd_inputs_check from bfabric_app_runner.cli.outputs import cmd_outputs_register, cmd_outputs_register_single_file @@ -87,6 +87,7 @@ cmd_prepare = cyclopts.App(name="prepare", help="Prepare a workunit for execution.") cmd_prepare.command(cmd_prepare_workunit, name="workunit") +cmd_prepare.command(cmd_prepare_local_workunit, name="local-workunit") cmd_prepare.group = groups["Running Apps"] app.command(cmd_prepare) diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/cmd_prepare.py b/bfabric_app_runner/src/bfabric_app_runner/cli/cmd_prepare.py index aa40d8b35..299691365 100644 --- a/bfabric_app_runner/src/bfabric_app_runner/cli/cmd_prepare.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/cmd_prepare.py @@ -4,18 +4,22 @@ import yaml from bfabric import Bfabric -from bfabric.experimental.workunit_definition import WorkunitDefinition +from bfabric.experimental.workunit_definition import WorkunitDefinition, WorkunitExecutionDefinition from bfabric.utils.cli_integration import use_client from bfabric_app_runner.actions.config_file import ActionConfig +from bfabric_app_runner.prepare.local_workunit_definition import LocalWorkunitExecutionDefinition from bfabric_app_runner.prepare.makefile_template import render_makefile from bfabric_app_runner.specs.app.app_spec import AppSpec -def _update_app_version(workunit_definition: WorkunitDefinition, application_version: str) -> WorkunitDefinition: +def _update_app_version[T: ( + WorkunitExecutionDefinition, + LocalWorkunitExecutionDefinition, +)](workunit_execution_definition: T, application_version: str) -> T: # TODO if this is useful consider moving it to the WorkunitDefinition class - workunit_definition = copy.deepcopy(workunit_definition) - workunit_definition.execution.raw_parameters["application_version"] = application_version - return workunit_definition + workunit_execution_definition = copy.deepcopy(workunit_execution_definition) + workunit_execution_definition.raw_parameters["application_version"] = application_version + return workunit_execution_definition @use_client @@ -46,8 +50,8 @@ def cmd_prepare_workunit( workunit_definition = WorkunitDefinition.from_ref(workunit_ref, client=client) if force_app_version: - workunit_definition = _update_app_version( - workunit_definition=workunit_definition, application_version=force_app_version + workunit_definition.execution = _update_app_version( + workunit_execution_definition=workunit_definition.execution, application_version=force_app_version ) # Resolve the app version early, following the pattern used by other commands @@ -76,6 +80,51 @@ def cmd_prepare_workunit( ) +def cmd_prepare_local_workunit( + app_spec: Path, + work_dir: Path, + workunit_ref: Path, + *, + ssh_user: str | None = None, + force_storage: Path | None = None, + force_app_version: str | None = None, + read_only: bool = False, + use_external_runner: bool = False, + # TODO these two are a bit weird but required right now (because they correspond to registration and not execution) + app_id: int = 0, + app_name: str = "local_app", +) -> None: + """Exprimental: Prepares a local workunit for processing.""" + # TODO this is mostly cmd_prepare_workunit with some changes + work_dir.mkdir(parents=True, exist_ok=True) + workunit_execution_definition = LocalWorkunitExecutionDefinition.from_yaml(workunit_ref) + if force_app_version: + workunit_execution_definition = _update_app_version( + workunit_execution_definition=workunit_execution_definition, application_version=force_app_version + ) + + # Resolve the app version early, following the pattern used by other commands + app_full_spec = AppSpec.load_yaml(app_yaml=app_spec, app_id=app_id, app_name=app_name) + + workunit_definition_path = work_dir / "workunit_definition.yml" + workunit_execution_definition.to_yaml(path=workunit_definition_path) + _write_app_env_file( + path=work_dir / "app_env.yml", + app_ref=app_spec.resolve(), + workunit_ref=workunit_definition_path, + ssh_user=ssh_user, + force_storage=force_storage, + read_only=read_only, + ) + # Render the workunit Makefile template + render_makefile( + path=work_dir / "Makefile", + bfabric_app_spec=app_full_spec.bfabric, + rename_existing=True, + use_external_runner=use_external_runner, + ) + + def _write_app_env_file( path: Path, app_ref: Path, diff --git a/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py b/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py index 6e18b79da..053bd8d5a 100644 --- a/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py +++ b/bfabric_app_runner/src/bfabric_app_runner/prepare/local_workunit_definition.py @@ -2,10 +2,16 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +import yaml from pydantic import BaseModel, model_validator from bfabric_app_runner.specs.inputs_spec import LocalInputSpecType # noqa: TC001 +if TYPE_CHECKING: + from pathlib import Path + class LocalWorkunitExecutionDefinition(BaseModel): """Experimental definition of a local workunit for local execution. @@ -28,3 +34,8 @@ def _mutually_exclusive(self) -> LocalWorkunitExecutionDefinition: if (self.dataset is None) == (not self.resources): raise ValueError("either dataset or resources must be provided, but not both") return self + + @classmethod + def from_yaml(cls, path: Path) -> LocalWorkunitExecutionDefinition: + parsed = yaml.safe_load(path.read_text()) + return cls.model_validate(parsed)