Upgrade pydantic v2

.github/workflows/codecov.yml

@@ -11,9 +11,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.13
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/conda_build.yml

@@ -13,7 +13,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         auto-update-conda: true
-        python-version: 3.9
+        python-version: 3.13
     - name: Build and upload conda package
       shell: bash -l {0}
       env:

.github/workflows/gh-pages.yml

@@ -10,9 +10,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: select python version
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.13
       - name: install dependencies
         run: |
           sudo apt-get update

.github/workflows/publish_to_pypi.yml

@@ -10,9 +10,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.13
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/pull_request_tests.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

jade/cli/common.py

@@ -164,7 +164,7 @@ def proceed_with_user_permission(ctx, message):
         "-m",
         "--resource-monitor-stats",
         multiple=True,
-        type=click.Choice([x for x in ResourceMonitorStats.__fields__]),
+        type=click.Choice([x for x in ResourceMonitorStats.model_fields]),
         help="Resource stats to monitor. Default is CPU and memory. "
         "Ex: -m cpu -m memory -m process",
     ),
@@ -334,7 +334,7 @@ def make_submitter_params(
         resource_monitor_stats = ResourceMonitorStats()
     else:
         stats = {x: True for x in resource_monitor_stats}
-        for field in ResourceMonitorStats.__fields__:
+        for field in ResourceMonitorStats.model_fields:
             if field not in stats:
                 stats[field] = False
         resource_monitor_stats = ResourceMonitorStats(**stats)

jade/cli/config.py

@@ -251,7 +251,7 @@ def hpc(account, config_file, mem, partition, qos, hpc_type, tmp, walltime):
         hpc = LocalHpcConfig()
 
     # This converts enums to values.
-    data = json.loads(HpcConfig(hpc_type=hpc_type, hpc=hpc).json())
+    data = json.loads(HpcConfig(hpc_type=hpc_type, hpc=hpc).model_dump_json())
     dump_data(data, config_file)
     print(f"Created HPC config file {config_file}")
 
@@ -478,7 +478,7 @@ def submitter_params(
         no_distributed_submitter=no_distributed_submitter,
     )
     # This converts enums to values.
-    data = json.loads(params.json())
+    data = json.loads(params.model_dump_json())
     if config_file.suffix == ".json":
         dump_data(data, config_file, indent=2)
     else:

jade/cli/spark.py

@@ -312,15 +312,15 @@ def config(
             sys.exit(1)
         config = load_data(update_config_file)
         for job in config["jobs"]:
-            job["spark_config"] = spark_config.dict()
+            job["spark_config"] = spark_config.model_dump()
         dump_data(config, update_config_file, indent=2)
         print(f"Updated jobs in {update_config_file} with this Spark configuration.")
     else:
         print(
             "\nAdd and customize this JSON object to the 'spark_config' field for each Spark "
             "job in your config.json file:\n"
         )
-        print(spark_config.json(indent=2))
+        print(spark_config.model_dump_json(indent=2))
 
 
 def _should_use_gpus(hpc_config, gpu):

jade/extensions/generic_command/generic_command_parameters.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Set
 
-from pydantic.v1 import Field, validator
+from pydantic import Field, field_validator, model_serializer
 
 from jade.models import JadeBaseModel
 from jade.models.spark import SparkConfigModel, SparkContainerModel
@@ -30,14 +30,14 @@ def __str__(self):
         return "<GenericCommandParameters: {}>".format(self.name)
 
     def __getattr__(self, name):
-        if name in GenericCommandParametersModel.__fields__:
+        if name in GenericCommandParametersModel.model_fields:
             return getattr(self._model, name)
         raise AttributeError(f"'GenericCommandParameters' object has no attribute '{name}'")
 
     def __setattr__(self, name, value):
         if name == "extension":
             raise AttributeError(f"'GenericCommandParameters' does not allow setting 'extension'")
-        if name in GenericCommandParametersModel.__fields__:
+        if name in GenericCommandParametersModel.model_fields:
             setattr(self._model, name, value)
         self.__dict__[name] = value
 
@@ -67,7 +67,7 @@ def _create_name(self):
     def serialize(self):
         assert self._model.job_id is not None
         # If job sizes get huge then we should exclude parameters with default values.
-        return self._model.dict()
+        return self._model.model_dump()
 
     @classmethod
     def deserialize(cls, data):
@@ -104,6 +104,7 @@ class GenericCommandParametersModel(JadeBaseModel):
     name: Optional[str] = Field(
         title="name",
         description="If not set Jade will use the job_id converted to a string. Must be unique.",
+        default=None,
     )
     use_multi_node_manager: bool = Field(
         title="use_multi_node_manager",
@@ -133,6 +134,7 @@ class GenericCommandParametersModel(JadeBaseModel):
         title="estimated_run_minutes",
         description="JADE will use this value along with num-parallel-processes-per-node and "
         "walltime to build per-node batches of jobs if time-based-batching is enabled.",
+        default=None,
     )
     submission_group: str = Field(
         title="submission_group",
@@ -159,39 +161,51 @@ class GenericCommandParametersModel(JadeBaseModel):
     job_id: Optional[int] = Field(
         title="job_id",
         description="Unique job identifier, generated by JADE",
+        default=None,
     )
     extension: str = Field(
         title="extension",
         description="job extension type, generated by JADE",
         default=_EXTENSION,
     )
 
-    @validator("append_output_dir")
-    def handle_append_output_dir(cls, value, values):
+    @field_validator("append_output_dir")
+    @classmethod
+    def handle_append_output_dir(cls, value, info):
         spark_enabled = False
-        if values["spark_config"] is not None:
-            spark_enabled = getattr(values["spark_config"], "enabled")
-        if values["use_multi_node_manager"] or spark_enabled:
+        if info.data["spark_config"] is not None:
+            spark_enabled = getattr(info.data["spark_config"], "enabled")
+        if info.data["use_multi_node_manager"] or spark_enabled:
             logger.debug(
                 "Override 'append_output_dir' because 'use_multi_node_manager' is set or spark is enabled"
             )
             return True
         return value
 
-    @validator("blocked_by")
+    @field_validator("blocked_by", mode="before")
+    @classmethod
     def handle_blocked_by(cls, value):
-        return {str(x) for x in value}
+        # Coerce sequence elements (e.g. integer job ids) to strings. Leave non-sequence input
+        # untouched so Pydantic raises its normal validation error instead of splitting a string.
+        if isinstance(value, (list, tuple, set, frozenset)):
+            return {str(x) for x in value}
+        return value
 
-    def dict(self, *args, **kwargs):
-        data = super().dict(*args, **kwargs)
-        # Keep the config file smaller by skipping values that are defaults.
+    @model_serializer(mode="wrap")
+    def _serialize(self, handler):
+        # Keep the config file smaller by skipping values that are defaults. Using a
+        # model_serializer ensures this applies across all Pydantic v2 serialization paths.
+        data = handler(self)
         for field in (
             "use_multi_node_manager",
             "spark_config",
             "append_job_name",
             "append_output_dir",
             "ext",
         ):
-            if data[field] == GenericCommandParametersModel.__fields__[field].default:
+            if (
+                field in data
+                and data[field] == GenericCommandParametersModel.model_fields[field].default
+            ):
                 data.pop(field)
         return data

jade/jobs/cluster.py

@@ -249,7 +249,7 @@ def get_status_summary(self, include_jobs=False):
         }
 
         if include_jobs:
-            summary["job_status"] = self._job_status.dict()
+            summary["job_status"] = self._job_status.model_dump()
 
         return summary
 
@@ -366,7 +366,7 @@ def serialize_submission_groups(self, directory):
 
         """
         path = directory / self.SUBMITTER_GROUP_FILE
-        data = [x.dict() for x in self._config.submission_groups]
+        data = [x.model_dump() for x in self._config.submission_groups]
         dump_data(data, path, cls=ExtendedJSONEncoder)
 
     @staticmethod
@@ -558,12 +558,12 @@ def _serialize(self, reason):
             )
 
         # Check the hash before the version update.
-        if hash(self._config.json()) != self._config_hash:
+        if hash(self._config.model_dump_json()) != self._config_hash:
             self._config.version += 1
             self._serialize_config_version()
-            text = self._config.json()
+            text = self._config.model_dump_json()
             self._config_hash = hash(text)
-            self._serialize_file(self._config.json(), self._config_file)
+            self._serialize_file(text, self._config_file)
             logger.info(
                 "Wrote config version %s reason=%s hostname=%s",
                 self._config.version,
@@ -579,10 +579,10 @@ def _serialize_jobs(self, reason):
             )
 
         # Check the hash before the version update.
-        if hash(self._job_status.json()) != self._job_status_hash:
+        if hash(self._job_status.model_dump_json()) != self._job_status_hash:
             self._job_status.version += 1
             self._serialize_job_status_version()
-            text = self._job_status.json()
+            text = self._job_status.model_dump_json()
             self._serialize_file(text, self._job_status_file)
             self._job_status_hash = hash(text)
             logger.info(

jade/jobs/job_configuration.py

@@ -293,7 +293,7 @@ def check_submission_groups(self):
         must_be_same = ("max_nodes", "poll_interval")
         all_params = (must_be_same, group_params, user_overrides, user_override_if_not_set)
         fields = {item for params in all_params for item in params}
-        assert sorted(list(fields)) == sorted(SubmitterParams.__fields__), sorted(list(fields))
+        assert sorted(list(fields)) == sorted(SubmitterParams.model_fields), sorted(list(fields))
         hpc_type = first_group.submitter_params.hpc_config.hpc_type
         group_names = set()
         for group in self.submission_groups:
@@ -579,7 +579,7 @@ def serialize(self, include=ConfigSerializeOptions.JOBS):
             "configuration_class": self.__class__.__name__,
             "format_version": self.FORMAT_VERSION,
             "user_data": self._user_data,
-            "submission_groups": [x.dict() for x in self.submission_groups],
+            "submission_groups": [x.model_dump() for x in self.submission_groups],
             "setup_command": self.setup_command,
             "teardown_command": self.teardown_command,
             "node_setup_command": self.node_setup_command,

jade/jobs/pipeline_manager.py

@@ -131,7 +131,7 @@ def create_config_from_files(config_files, pipeline_config_file, submit_params):
 
         config = PipelineConfig(stages=stages, stage_num=1)
         with open(pipeline_config_file, "w") as f_out:
-            f_out.write(config.json(indent=2))
+            f_out.write(config.model_dump_json(indent=2))
         logger.info("Created pipeline config file %s", pipeline_config_file)
 
     @staticmethod
@@ -164,16 +164,15 @@ def create_config_from_commands(auto_config_cmds, pipeline_config_file, submit_p
 
         config = PipelineConfig(stages=stages, stage_num=1)
         with open(pipeline_config_file, "w") as f_out:
-            f_out.write(config.json(indent=2))
+            f_out.write(config.model_dump_json(indent=2))
         logger.info("Created pipeline config file %s", pipeline_config_file)
 
     def _deserialize(self):
         return PipelineConfig(**load_data(self._config_file))
 
     def _serialize(self):
-        print(self.stage_num)
         with open(self._config_file, "w") as f_out:
-            f_out.write(self._config.json(indent=2))
+            f_out.write(self._config.model_dump_json(indent=2))
 
     def _submit_next_stage(self, stage_num, return_code=None):
         if return_code is None:

jade/models/__init__.py

@@ -1,4 +1,4 @@
-from pydantic.v1 import BaseModel
+from pydantic import BaseModel
 
 from jade.models.base import JadeBaseModel
 from jade.models.hpc import HpcConfig, SlurmConfig, FakeHpcConfig, LocalHpcConfig
@@ -21,5 +21,6 @@ def get_model_defaults(model_class: BaseModel):
 
     """
     return {
-        x: y.get("default") for x, y in model_class.schema(by_alias=False)["properties"].items()
+        x: y.get("default")
+        for x, y in model_class.model_json_schema(by_alias=False)["properties"].items()
     }

jade/models/base.py

@@ -2,22 +2,23 @@
 
 from pathlib import Path
 
-from pydantic.v1 import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from jade.utils.utils import load_data
 
 
 class JadeBaseModel(BaseModel):
     """Base class for JADE models."""
 
-    class Config:
-        title = "JadeBaseModel"
-        anystr_strip_whitespace = True
-        validate_assignment = True
-        validate_all = True
-        extra = "forbid"
-        use_enum_values = False
-        allow_population_by_field_name = True
+    model_config = ConfigDict(
+        title="JadeBaseModel",
+        str_strip_whitespace=True,
+        validate_assignment=True,
+        validate_default=True,
+        extra="forbid",
+        use_enum_values=False,
+        populate_by_name=True,
+    )
 
     @classmethod
     def load(cls, path: Path):

jade/models/cluster_config.py

@@ -2,7 +2,7 @@
 
 from typing import List, Optional
 
-from pydantic.v1 import Field
+from pydantic import Field
 
 from jade.models import JadeBaseModel, SubmissionGroup
 
@@ -26,6 +26,7 @@ class ClusterConfig(JadeBaseModel):
     pipeline_stage_num: Optional[int] = Field(
         title="pipeline_stage_num",
         description="stage number if the config is part of a pipeline",
+        default=None,
     )
     num_jobs: int = Field(
         title="num_jobs",