From 97cd5fa55d89f0b40203843b70d04b3efd75fe65 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 16 Jun 2026 08:42:55 -0600 Subject: [PATCH 1/4] Use Python 3.13 in actions --- .github/workflows/codecov.yml | 2 +- .github/workflows/conda_build.yml | 2 +- .github/workflows/gh-pages.yml | 2 +- .github/workflows/publish_to_pypi.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 751fc010..7bbe904b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.13 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/conda_build.yml b/.github/workflows/conda_build.yml index 95976070..56e27749 100644 --- a/.github/workflows/conda_build.yml +++ b/.github/workflows/conda_build.yml @@ -13,7 +13,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: 3.9 + python-version: 3.13 - name: Build and upload conda package shell: bash -l {0} env: diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 45fd85ce..db47407c 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -12,7 +12,7 @@ jobs: - name: select python version uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.13 - name: install dependencies run: | sudo apt-get update diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index d1ecef30..ebaecd80 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.13 - name: Install dependencies run: | python -m pip install --upgrade pip From e96e4d19b9293feefe4a8100af62f5fe017c6467 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 16 Jun 2026 08:43:22 -0600 Subject: [PATCH 2/4] Upgrade to pydantic v2 models --- jade/cli/common.py | 4 +- jade/cli/config.py | 4 +- jade/cli/spark.py | 4 +- .../generic_command_parameters.py | 31 ++++++++------ jade/jobs/cluster.py | 14 +++---- jade/jobs/job_configuration.py | 4 +- jade/jobs/pipeline_manager.py | 6 +-- jade/models/__init__.py | 5 ++- jade/models/base.py | 19 +++++---- jade/models/cluster_config.py | 3 +- jade/models/hpc.py | 41 ++++++++++--------- jade/models/jobs.py | 7 +++- jade/models/pipeline.py | 6 ++- jade/models/singularity.py | 10 +++-- jade/models/spark.py | 16 ++++---- jade/models/submission_group.py | 2 +- jade/models/submitter_params.py | 8 ++-- jade/utils/utils.py | 4 +- tests/test_hpc.py | 4 +- 19 files changed, 107 insertions(+), 85 deletions(-) diff --git a/jade/cli/common.py b/jade/cli/common.py index 1f0ad06b..74ba40c3 100644 --- a/jade/cli/common.py +++ b/jade/cli/common.py @@ -164,7 +164,7 @@ def proceed_with_user_permission(ctx, message): "-m", "--resource-monitor-stats", multiple=True, - type=click.Choice([x for x in ResourceMonitorStats.__fields__]), + type=click.Choice([x for x in ResourceMonitorStats.model_fields]), help="Resource stats to monitor. Default is CPU and memory. " "Ex: -m cpu -m memory -m process", ), @@ -334,7 +334,7 @@ def make_submitter_params( resource_monitor_stats = ResourceMonitorStats() else: stats = {x: True for x in resource_monitor_stats} - for field in ResourceMonitorStats.__fields__: + for field in ResourceMonitorStats.model_fields: if field not in stats: stats[field] = False resource_monitor_stats = ResourceMonitorStats(**stats) diff --git a/jade/cli/config.py b/jade/cli/config.py index 19498349..3a4bf7f7 100644 --- a/jade/cli/config.py +++ b/jade/cli/config.py @@ -251,7 +251,7 @@ def hpc(account, config_file, mem, partition, qos, hpc_type, tmp, walltime): hpc = LocalHpcConfig() # This converts enums to values. - data = json.loads(HpcConfig(hpc_type=hpc_type, hpc=hpc).json()) + data = json.loads(HpcConfig(hpc_type=hpc_type, hpc=hpc).model_dump_json()) dump_data(data, config_file) print(f"Created HPC config file {config_file}") @@ -478,7 +478,7 @@ def submitter_params( no_distributed_submitter=no_distributed_submitter, ) # This converts enums to values. - data = json.loads(params.json()) + data = json.loads(params.model_dump_json()) if config_file.suffix == ".json": dump_data(data, config_file, indent=2) else: diff --git a/jade/cli/spark.py b/jade/cli/spark.py index c86de5c2..64fc49e8 100644 --- a/jade/cli/spark.py +++ b/jade/cli/spark.py @@ -312,7 +312,7 @@ def config( sys.exit(1) config = load_data(update_config_file) for job in config["jobs"]: - job["spark_config"] = spark_config.dict() + job["spark_config"] = spark_config.model_dump() dump_data(config, update_config_file, indent=2) print(f"Updated jobs in {update_config_file} with this Spark configuration.") else: @@ -320,7 +320,7 @@ def config( "\nAdd and customize this JSON object to the 'spark_config' field for each Spark " "job in your config.json file:\n" ) - print(spark_config.json(indent=2)) + print(spark_config.model_dump_json(indent=2)) def _should_use_gpus(hpc_config, gpu): diff --git a/jade/extensions/generic_command/generic_command_parameters.py b/jade/extensions/generic_command/generic_command_parameters.py index 9b0b37b4..882dcb2d 100644 --- a/jade/extensions/generic_command/generic_command_parameters.py +++ b/jade/extensions/generic_command/generic_command_parameters.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict, List, Optional, Set -from pydantic.v1 import Field, validator +from pydantic import Field, field_validator from jade.models import JadeBaseModel from jade.models.spark import SparkConfigModel, SparkContainerModel @@ -30,14 +30,14 @@ def __str__(self): return "".format(self.name) def __getattr__(self, name): - if name in GenericCommandParametersModel.__fields__: + if name in GenericCommandParametersModel.model_fields: return getattr(self._model, name) raise AttributeError(f"'GenericCommandParameters' object has no attribute '{name}'") def __setattr__(self, name, value): if name == "extension": raise AttributeError(f"'GenericCommandParameters' does not allow setting 'extension'") - if name in GenericCommandParametersModel.__fields__: + if name in GenericCommandParametersModel.model_fields: setattr(self._model, name, value) self.__dict__[name] = value @@ -67,7 +67,7 @@ def _create_name(self): def serialize(self): assert self._model.job_id is not None # If job sizes get huge then we should exclude parameters with default values. - return self._model.dict() + return self._model.model_dump() @classmethod def deserialize(cls, data): @@ -104,6 +104,7 @@ class GenericCommandParametersModel(JadeBaseModel): name: Optional[str] = Field( title="name", description="If not set Jade will use the job_id converted to a string. Must be unique.", + default=None, ) use_multi_node_manager: bool = Field( title="use_multi_node_manager", @@ -133,6 +134,7 @@ class GenericCommandParametersModel(JadeBaseModel): title="estimated_run_minutes", description="JADE will use this value along with num-parallel-processes-per-node and " "walltime to build per-node batches of jobs if time-based-batching is enabled.", + default=None, ) submission_group: str = Field( title="submission_group", @@ -159,6 +161,7 @@ class GenericCommandParametersModel(JadeBaseModel): job_id: Optional[int] = Field( title="job_id", description="Unique job identifier, generated by JADE", + default=None, ) extension: str = Field( title="extension", @@ -166,24 +169,26 @@ class GenericCommandParametersModel(JadeBaseModel): default=_EXTENSION, ) - @validator("append_output_dir") - def handle_append_output_dir(cls, value, values): + @field_validator("append_output_dir") + @classmethod + def handle_append_output_dir(cls, value, info): spark_enabled = False - if values["spark_config"] is not None: - spark_enabled = getattr(values["spark_config"], "enabled") - if values["use_multi_node_manager"] or spark_enabled: + if info.data["spark_config"] is not None: + spark_enabled = getattr(info.data["spark_config"], "enabled") + if info.data["use_multi_node_manager"] or spark_enabled: logger.debug( "Override 'append_output_dir' because 'use_multi_node_manager' is set or spark is enabled" ) return True return value - @validator("blocked_by") + @field_validator("blocked_by", mode="before") + @classmethod def handle_blocked_by(cls, value): return {str(x) for x in value} - def dict(self, *args, **kwargs): - data = super().dict(*args, **kwargs) + def model_dump(self, *args, **kwargs): + data = super().model_dump(*args, **kwargs) # Keep the config file smaller by skipping values that are defaults. for field in ( "use_multi_node_manager", @@ -192,6 +197,6 @@ def dict(self, *args, **kwargs): "append_output_dir", "ext", ): - if data[field] == GenericCommandParametersModel.__fields__[field].default: + if data[field] == GenericCommandParametersModel.model_fields[field].default: data.pop(field) return data diff --git a/jade/jobs/cluster.py b/jade/jobs/cluster.py index 5201aa05..568a6194 100644 --- a/jade/jobs/cluster.py +++ b/jade/jobs/cluster.py @@ -249,7 +249,7 @@ def get_status_summary(self, include_jobs=False): } if include_jobs: - summary["job_status"] = self._job_status.dict() + summary["job_status"] = self._job_status.model_dump() return summary @@ -366,7 +366,7 @@ def serialize_submission_groups(self, directory): """ path = directory / self.SUBMITTER_GROUP_FILE - data = [x.dict() for x in self._config.submission_groups] + data = [x.model_dump() for x in self._config.submission_groups] dump_data(data, path, cls=ExtendedJSONEncoder) @staticmethod @@ -558,12 +558,12 @@ def _serialize(self, reason): ) # Check the hash before the version update. - if hash(self._config.json()) != self._config_hash: + if hash(self._config.model_dump_json()) != self._config_hash: self._config.version += 1 self._serialize_config_version() - text = self._config.json() + text = self._config.model_dump_json() self._config_hash = hash(text) - self._serialize_file(self._config.json(), self._config_file) + self._serialize_file(self._config.model_dump_json(), self._config_file) logger.info( "Wrote config version %s reason=%s hostname=%s", self._config.version, @@ -579,10 +579,10 @@ def _serialize_jobs(self, reason): ) # Check the hash before the version update. - if hash(self._job_status.json()) != self._job_status_hash: + if hash(self._job_status.model_dump_json()) != self._job_status_hash: self._job_status.version += 1 self._serialize_job_status_version() - text = self._job_status.json() + text = self._job_status.model_dump_json() self._serialize_file(text, self._job_status_file) self._job_status_hash = hash(text) logger.info( diff --git a/jade/jobs/job_configuration.py b/jade/jobs/job_configuration.py index cf858f26..5b30e853 100644 --- a/jade/jobs/job_configuration.py +++ b/jade/jobs/job_configuration.py @@ -293,7 +293,7 @@ def check_submission_groups(self): must_be_same = ("max_nodes", "poll_interval") all_params = (must_be_same, group_params, user_overrides, user_override_if_not_set) fields = {item for params in all_params for item in params} - assert sorted(list(fields)) == sorted(SubmitterParams.__fields__), sorted(list(fields)) + assert sorted(list(fields)) == sorted(SubmitterParams.model_fields), sorted(list(fields)) hpc_type = first_group.submitter_params.hpc_config.hpc_type group_names = set() for group in self.submission_groups: @@ -579,7 +579,7 @@ def serialize(self, include=ConfigSerializeOptions.JOBS): "configuration_class": self.__class__.__name__, "format_version": self.FORMAT_VERSION, "user_data": self._user_data, - "submission_groups": [x.dict() for x in self.submission_groups], + "submission_groups": [x.model_dump() for x in self.submission_groups], "setup_command": self.setup_command, "teardown_command": self.teardown_command, "node_setup_command": self.node_setup_command, diff --git a/jade/jobs/pipeline_manager.py b/jade/jobs/pipeline_manager.py index 41200e27..37394b6f 100644 --- a/jade/jobs/pipeline_manager.py +++ b/jade/jobs/pipeline_manager.py @@ -131,7 +131,7 @@ def create_config_from_files(config_files, pipeline_config_file, submit_params): config = PipelineConfig(stages=stages, stage_num=1) with open(pipeline_config_file, "w") as f_out: - f_out.write(config.json(indent=2)) + f_out.write(config.model_dump_json(indent=2)) logger.info("Created pipeline config file %s", pipeline_config_file) @staticmethod @@ -164,7 +164,7 @@ def create_config_from_commands(auto_config_cmds, pipeline_config_file, submit_p config = PipelineConfig(stages=stages, stage_num=1) with open(pipeline_config_file, "w") as f_out: - f_out.write(config.json(indent=2)) + f_out.write(config.model_dump_json(indent=2)) logger.info("Created pipeline config file %s", pipeline_config_file) def _deserialize(self): @@ -173,7 +173,7 @@ def _deserialize(self): def _serialize(self): print(self.stage_num) with open(self._config_file, "w") as f_out: - f_out.write(self._config.json(indent=2)) + f_out.write(self._config.model_dump_json(indent=2)) def _submit_next_stage(self, stage_num, return_code=None): if return_code is None: diff --git a/jade/models/__init__.py b/jade/models/__init__.py index 8b5f90f5..eac52e8d 100644 --- a/jade/models/__init__.py +++ b/jade/models/__init__.py @@ -1,4 +1,4 @@ -from pydantic.v1 import BaseModel +from pydantic import BaseModel from jade.models.base import JadeBaseModel from jade.models.hpc import HpcConfig, SlurmConfig, FakeHpcConfig, LocalHpcConfig @@ -21,5 +21,6 @@ def get_model_defaults(model_class: BaseModel): """ return { - x: y.get("default") for x, y in model_class.schema(by_alias=False)["properties"].items() + x: y.get("default") + for x, y in model_class.model_json_schema(by_alias=False)["properties"].items() } diff --git a/jade/models/base.py b/jade/models/base.py index 75214274..d14dc939 100644 --- a/jade/models/base.py +++ b/jade/models/base.py @@ -2,7 +2,7 @@ from pathlib import Path -from pydantic.v1 import BaseModel +from pydantic import BaseModel, ConfigDict from jade.utils.utils import load_data @@ -10,14 +10,15 @@ class JadeBaseModel(BaseModel): """Base class for JADE models.""" - class Config: - title = "JadeBaseModel" - anystr_strip_whitespace = True - validate_assignment = True - validate_all = True - extra = "forbid" - use_enum_values = False - allow_population_by_field_name = True + model_config = ConfigDict( + title="JadeBaseModel", + str_strip_whitespace=True, + validate_assignment=True, + validate_default=True, + extra="forbid", + use_enum_values=False, + populate_by_name=True, + ) @classmethod def load(cls, path: Path): diff --git a/jade/models/cluster_config.py b/jade/models/cluster_config.py index 3647b8a7..7c080f49 100644 --- a/jade/models/cluster_config.py +++ b/jade/models/cluster_config.py @@ -2,7 +2,7 @@ from typing import List, Optional -from pydantic.v1 import Field +from pydantic import Field from jade.models import JadeBaseModel, SubmissionGroup @@ -26,6 +26,7 @@ class ClusterConfig(JadeBaseModel): pipeline_stage_num: Optional[int] = Field( title="pipeline_stage_num", description="stage number if the config is part of a pipeline", + default=None, ) num_jobs: int = Field( title="num_jobs", diff --git a/jade/models/hpc.py b/jade/models/hpc.py index 7726986b..499a3fdb 100644 --- a/jade/models/hpc.py +++ b/jade/models/hpc.py @@ -3,7 +3,7 @@ import re from typing import Optional, Union, List -from pydantic.v1 import Field, validator, root_validator, validator +from pydantic import Field, field_validator, model_validator from jade.hpc.common import HpcType from jade.models.base import JadeBaseModel @@ -39,6 +39,7 @@ class SlurmConfig(JadeBaseModel): gres: Optional[str] = Field( title="gpu", description="Request nodes that have at least this number of GPUs. Ex: 'gpu:2'", + default=None, ) mem: Optional[str] = Field( title="mem", @@ -66,7 +67,8 @@ class SlurmConfig(JadeBaseModel): default=None, ) - @validator("gres") + @field_validator("gres") + @classmethod def check_gpus(cls, gres): if gres is None: return gres @@ -76,21 +78,18 @@ def check_gpus(cls, gres): ) return gres - @root_validator(pre=True) - def handle_allocation(cls, values: dict) -> dict: - if "allocation" in values: + @model_validator(mode="before") + @classmethod + def handle_allocation(cls, values): + if isinstance(values, dict) and "allocation" in values: values["account"] = values.pop("allocation") return values - @root_validator - def handle_nodes_and_tasks(cls, values: dict) -> dict: - if ( - values["nodes"] is None - and values["ntasks"] is None - and values["ntasks_per_node"] is None - ): - values["nodes"] = 1 - return values + @model_validator(mode="after") + def handle_nodes_and_tasks(self): + if self.nodes is None and self.ntasks is None and self.ntasks_per_node is None: + self.nodes = 1 + return self class FakeHpcConfig(JadeBaseModel): @@ -124,18 +123,20 @@ class HpcConfig(JadeBaseModel): description="Interface-specific config options", ) - @validator("hpc", pre=True) - def assign_hpc(cls, value, values): + @field_validator("hpc", mode="before") + @classmethod + def assign_hpc(cls, value, info): if isinstance(value, JadeBaseModel): return value - if values["hpc_type"] == HpcType.SLURM: + hpc_type = info.data["hpc_type"] + if hpc_type == HpcType.SLURM: return SlurmConfig(**value) - elif values["hpc_type"] == HpcType.FAKE: + elif hpc_type == HpcType.FAKE: return FakeHpcConfig(**value) - elif values["hpc_type"] == HpcType.LOCAL: + elif hpc_type == HpcType.LOCAL: return LocalHpcConfig() - raise ValueError(f"Unsupported: {values['hpc_type']}") + raise ValueError(f"Unsupported: {hpc_type}") def get_num_gpus(self): """Return the number of GPUs specified by the config. diff --git a/jade/models/jobs.py b/jade/models/jobs.py index bf82f692..301ded7f 100644 --- a/jade/models/jobs.py +++ b/jade/models/jobs.py @@ -3,7 +3,7 @@ import enum from typing import List, Optional, Set, Union -from pydantic.v1 import Field +from pydantic import Field, field_validator from jade.models.base import JadeBaseModel @@ -58,3 +58,8 @@ class JobStatus(JadeBaseModel): title="version", description="version of the statuses, increments with each update", ) + + @field_validator("hpc_job_ids", mode="before") + @classmethod + def coerce_hpc_job_ids(cls, value): + return [str(x) for x in value] diff --git a/jade/models/pipeline.py b/jade/models/pipeline.py index 78f25f97..9cd8e623 100644 --- a/jade/models/pipeline.py +++ b/jade/models/pipeline.py @@ -2,7 +2,7 @@ from typing import List, Optional -from pydantic.v1 import Field +from pydantic import Field from jade.models import JadeBaseModel, SubmitterParams @@ -13,6 +13,7 @@ class PipelineStage(JadeBaseModel): auto_config_cmd: Optional[str] = Field( title="auto_config_cmd", description="command used to create the JADE configuration", + default=None, ) config_file: str = Field( title="config_file", @@ -25,10 +26,12 @@ class PipelineStage(JadeBaseModel): path: Optional[str] = Field( title="path", description="directory on shared filesystem containing config", + default=None, ) return_code: Optional[int] = Field( title="return_code", description="return code of stage; 0 is success", + default=None, ) submitter_params: SubmitterParams = Field( title="submitter_params", @@ -42,6 +45,7 @@ class PipelineConfig(JadeBaseModel): path: Optional[str] = Field( title="path", description="directory on shared filesystem containing config", + default=None, ) stage_num: int = Field( title="stage_num", diff --git a/jade/models/singularity.py b/jade/models/singularity.py index 62de81ce..e75d2c74 100644 --- a/jade/models/singularity.py +++ b/jade/models/singularity.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Optional -from pydantic.v1 import Field, validator +from pydantic import Field, field_validator from jade.models import JadeBaseModel @@ -29,6 +29,7 @@ class SingularityParams(JadeBaseModel): container: Optional[str] = Field( title="container", description="Path to Singularity container", + default=None, ) load_command: str = Field( title="load_command", @@ -46,9 +47,10 @@ class SingularityParams(JadeBaseModel): default=SINGULARITY_SETUP_COMMANDS, ) - @validator("container") - def check_container(cls, container, values): - if values["enabled"]: + @field_validator("container") + @classmethod + def check_container(cls, container, info): + if info.data["enabled"]: if container is None: raise ValueError("'container' must be set") if not Path(container).exists(): diff --git a/jade/models/spark.py b/jade/models/spark.py index 213dac94..fe2cf556 100644 --- a/jade/models/spark.py +++ b/jade/models/spark.py @@ -1,7 +1,7 @@ from pathlib import Path -from typing import Dict, List, Set +from typing import Dict, List, Optional, Set -from pydantic.v1 import Field, root_validator +from pydantic import Field, model_validator from jade.models import JadeBaseModel @@ -57,7 +57,7 @@ class SparkConfigModel(JadeBaseModel): description="Use node's tmpfs instead of internal storage for scratch space.", default=False, ) - alt_scratch: str = Field( + alt_scratch: Optional[str] = Field( title="alt_scratch", description="Use this alternative directory for scratch space.", default=None, @@ -68,11 +68,13 @@ class SparkConfigModel(JadeBaseModel): default=0, ) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def handle_legacy_values(cls, values): - run_outside = values.pop("run_user_script_outside_container", None) - if run_outside is not None and "run_user_script_inside_container" not in values: - values["run_user_script_inside_container"] = not run_outside + if isinstance(values, dict): + run_outside = values.pop("run_user_script_outside_container", None) + if run_outside is not None and "run_user_script_inside_container" not in values: + values["run_user_script_inside_container"] = not run_outside return values def get_spark_script(self): diff --git a/jade/models/submission_group.py b/jade/models/submission_group.py index cafe13b9..f3d97eaf 100644 --- a/jade/models/submission_group.py +++ b/jade/models/submission_group.py @@ -1,6 +1,6 @@ """Defines parameters for submitting jobs to an HPC.""" -from pydantic.v1 import Field +from pydantic import Field from jade.models import JadeBaseModel, SubmitterParams diff --git a/jade/models/submitter_params.py b/jade/models/submitter_params.py index 0bfae0e8..ae1e4ce1 100644 --- a/jade/models/submitter_params.py +++ b/jade/models/submitter_params.py @@ -4,7 +4,7 @@ from datetime import timedelta from typing import Optional -from pydantic.v1 import Field +from pydantic import Field from jade.enums import ResourceMonitorType from jade.models import JadeBaseModel, HpcConfig, SingularityParams @@ -75,7 +75,7 @@ class SubmitterParams(JadeBaseModel): description="Script to run on each node after completing jobs", default=None, ) - poll_interval: int = Field( + poll_interval: float = Field( description="Interval in seconds on which to poll jobs for status", default=10, ) @@ -132,8 +132,8 @@ def get_wall_time(self): return timedelta(seconds=0xFFFFFFFF) # largest 8-byte integer return _to_timedelta(wall_time) - def dict(self, *args, **kwargs): - data = super().dict(*args, **kwargs) + def model_dump(self, *args, **kwargs): + data = super().model_dump(*args, **kwargs) if data["node_setup_script"] is None: data.pop("node_setup_script") if data["node_shutdown_script"] is None: diff --git a/jade/utils/utils.py b/jade/utils/utils.py index d8c92817..9922c03d 100644 --- a/jade/utils/utils.py +++ b/jade/utils/utils.py @@ -16,7 +16,7 @@ from dateutil.parser import parse import toml -from pydantic.v1 import BaseModel +from pydantic import BaseModel from jade.exceptions import InvalidParameter from jade.utils.timing_utils import timed_debug @@ -518,6 +518,6 @@ def default(self, obj): return list(obj) if isinstance(obj, BaseModel): - return obj.dict() + return obj.model_dump() return json.JSONEncoder.default(self, obj) diff --git a/tests/test_hpc.py b/tests/test_hpc.py index 77ba46c6..19e9d88a 100644 --- a/tests/test_hpc.py +++ b/tests/test_hpc.py @@ -2,7 +2,7 @@ import os -from pydantic.v1.error_wrappers import ValidationError +from pydantic import ValidationError import pytest from jade.common import OUTPUT_DIR @@ -52,7 +52,7 @@ def hpc_config(hpc_type, **kwargs): def test_create_slurm(): create_hpc_manager("slurm") config = hpc_config("slurm") - bad_config = config.dict() + bad_config = config.model_dump() bad_config["hpc"].pop("account") with pytest.raises(ValidationError): HpcConfig(**bad_config) From 42b7129266cca184b1df2fef85743d94667db983 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 16 Jun 2026 10:57:24 -0600 Subject: [PATCH 3/4] Address PR comments --- .../generic_command/generic_command_parameters.py | 5 ++++- jade/jobs/cluster.py | 2 +- jade/jobs/pipeline_manager.py | 1 - jade/models/submitter_params.py | 8 ++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jade/extensions/generic_command/generic_command_parameters.py b/jade/extensions/generic_command/generic_command_parameters.py index 882dcb2d..d69bd3a6 100644 --- a/jade/extensions/generic_command/generic_command_parameters.py +++ b/jade/extensions/generic_command/generic_command_parameters.py @@ -197,6 +197,9 @@ def model_dump(self, *args, **kwargs): "append_output_dir", "ext", ): - if data[field] == GenericCommandParametersModel.model_fields[field].default: + if ( + field in data + and data[field] == GenericCommandParametersModel.model_fields[field].default + ): data.pop(field) return data diff --git a/jade/jobs/cluster.py b/jade/jobs/cluster.py index 568a6194..5cbf3b0b 100644 --- a/jade/jobs/cluster.py +++ b/jade/jobs/cluster.py @@ -563,7 +563,7 @@ def _serialize(self, reason): self._serialize_config_version() text = self._config.model_dump_json() self._config_hash = hash(text) - self._serialize_file(self._config.model_dump_json(), self._config_file) + self._serialize_file(text, self._config_file) logger.info( "Wrote config version %s reason=%s hostname=%s", self._config.version, diff --git a/jade/jobs/pipeline_manager.py b/jade/jobs/pipeline_manager.py index 37394b6f..2f1584d8 100644 --- a/jade/jobs/pipeline_manager.py +++ b/jade/jobs/pipeline_manager.py @@ -171,7 +171,6 @@ def _deserialize(self): return PipelineConfig(**load_data(self._config_file)) def _serialize(self): - print(self.stage_num) with open(self._config_file, "w") as f_out: f_out.write(self._config.model_dump_json(indent=2)) diff --git a/jade/models/submitter_params.py b/jade/models/submitter_params.py index ae1e4ce1..bf8b7eb8 100644 --- a/jade/models/submitter_params.py +++ b/jade/models/submitter_params.py @@ -134,10 +134,10 @@ def get_wall_time(self): def model_dump(self, *args, **kwargs): data = super().model_dump(*args, **kwargs) - if data["node_setup_script"] is None: - data.pop("node_setup_script") - if data["node_shutdown_script"] is None: - data.pop("node_shutdown_script") + if data.get("node_setup_script") is None: + data.pop("node_setup_script", None) + if data.get("node_shutdown_script") is None: + data.pop("node_shutdown_script", None) return data From 0b02ef0cfb16db7c93939b1eb8f5070d4d9c1c32 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 16 Jun 2026 11:30:17 -0600 Subject: [PATCH 4/4] Address second round of PR comments - Use @model_serializer instead of overriding model_dump() so obsolete/ default fields are dropped across model_dump_json() and nested serialization (SubmitterParams, GenericCommandParametersModel) - Guard before-validators so a plain string is rejected rather than split into characters (blocked_by, hpc_job_ids) - assign_hpc: use info.data.get("hpc_type") for a structured ValidationError instead of a raw KeyError - Fix misleading gres validation message ('gpu:N', not 'gres=gpu:N') - Bump actions/setup-python@v2 -> @v5 for reliable Python 3.13 provisioning Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/codecov.yml | 2 +- .github/workflows/gh-pages.yml | 2 +- .github/workflows/publish_to_pypi.yml | 2 +- .github/workflows/pull_request_tests.yml | 2 +- .../generic_command_parameters.py | 16 +++++++++++----- jade/models/hpc.py | 6 ++++-- jade/models/jobs.py | 6 +++++- jade/models/submitter_params.py | 10 +++++++--- 8 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 7bbe904b..8667073d 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.13 - name: Install dependencies diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index db47407c..35728628 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: select python version - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.13 - name: install dependencies diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index ebaecd80..f7740e41 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.13 - name: Install dependencies diff --git a/.github/workflows/pull_request_tests.yml b/.github/workflows/pull_request_tests.yml index c02f9c06..a8e8a9d0 100644 --- a/.github/workflows/pull_request_tests.yml +++ b/.github/workflows/pull_request_tests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/jade/extensions/generic_command/generic_command_parameters.py b/jade/extensions/generic_command/generic_command_parameters.py index d69bd3a6..341a984f 100644 --- a/jade/extensions/generic_command/generic_command_parameters.py +++ b/jade/extensions/generic_command/generic_command_parameters.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict, List, Optional, Set -from pydantic import Field, field_validator +from pydantic import Field, field_validator, model_serializer from jade.models import JadeBaseModel from jade.models.spark import SparkConfigModel, SparkContainerModel @@ -185,11 +185,17 @@ def handle_append_output_dir(cls, value, info): @field_validator("blocked_by", mode="before") @classmethod def handle_blocked_by(cls, value): - return {str(x) for x in value} + # Coerce sequence elements (e.g. integer job ids) to strings. Leave non-sequence input + # untouched so Pydantic raises its normal validation error instead of splitting a string. + if isinstance(value, (list, tuple, set, frozenset)): + return {str(x) for x in value} + return value - def model_dump(self, *args, **kwargs): - data = super().model_dump(*args, **kwargs) - # Keep the config file smaller by skipping values that are defaults. + @model_serializer(mode="wrap") + def _serialize(self, handler): + # Keep the config file smaller by skipping values that are defaults. Using a + # model_serializer ensures this applies across all Pydantic v2 serialization paths. + data = handler(self) for field in ( "use_multi_node_manager", "spark_config", diff --git a/jade/models/hpc.py b/jade/models/hpc.py index 499a3fdb..17b1e219 100644 --- a/jade/models/hpc.py +++ b/jade/models/hpc.py @@ -74,7 +74,7 @@ def check_gpus(cls, gres): return gres if re.search(r"^gpu:(\d+)$", gres) is None: raise ValueError( - "gres value must follow the format 'gres=gpu:N' where N is the number of required GPUs" + "gres value must follow the format 'gpu:N' where N is the number of required GPUs" ) return gres @@ -129,7 +129,9 @@ def assign_hpc(cls, value, info): if isinstance(value, JadeBaseModel): return value - hpc_type = info.data["hpc_type"] + hpc_type = info.data.get("hpc_type") + if hpc_type is None: + raise ValueError("'hpc_type' is required to validate 'hpc'") if hpc_type == HpcType.SLURM: return SlurmConfig(**value) elif hpc_type == HpcType.FAKE: diff --git a/jade/models/jobs.py b/jade/models/jobs.py index 301ded7f..06baea17 100644 --- a/jade/models/jobs.py +++ b/jade/models/jobs.py @@ -62,4 +62,8 @@ class JobStatus(JadeBaseModel): @field_validator("hpc_job_ids", mode="before") @classmethod def coerce_hpc_job_ids(cls, value): - return [str(x) for x in value] + # Coerce sequence elements (e.g. integer job ids) to strings. Leave non-sequence input + # untouched so Pydantic raises its normal validation error instead of splitting a string. + if isinstance(value, (list, tuple, set, frozenset)): + return [str(x) for x in value] + return value diff --git a/jade/models/submitter_params.py b/jade/models/submitter_params.py index bf8b7eb8..0d3d2d47 100644 --- a/jade/models/submitter_params.py +++ b/jade/models/submitter_params.py @@ -4,7 +4,7 @@ from datetime import timedelta from typing import Optional -from pydantic import Field +from pydantic import Field, model_serializer from jade.enums import ResourceMonitorType from jade.models import JadeBaseModel, HpcConfig, SingularityParams @@ -132,8 +132,12 @@ def get_wall_time(self): return timedelta(seconds=0xFFFFFFFF) # largest 8-byte integer return _to_timedelta(wall_time) - def model_dump(self, *args, **kwargs): - data = super().model_dump(*args, **kwargs) + @model_serializer(mode="wrap") + def _serialize(self, handler): + # Drop the obsolete script fields when unset. Using a model_serializer (rather than + # overriding model_dump) ensures this applies across model_dump, model_dump_json, and + # nested serialization in Pydantic v2. + data = handler(self) if data.get("node_setup_script") is None: data.pop("node_setup_script", None) if data.get("node_shutdown_script") is None: