Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mlpstorage
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#! /bin/bash
uv run python3 -m mlpstorage_py.main $*
19 changes: 0 additions & 19 deletions mlpstorage/rules/run_checkers/__init__.py

This file was deleted.

1 change: 0 additions & 1 deletion mlpstorage/tests/__init__.py

This file was deleted.

6 changes: 3 additions & 3 deletions mlpstorage/__init__.py → mlpstorage_py/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from importlib.metadata import version as _pkg_version, PackageNotFoundError as _PkgNF
try:
VERSION = _pkg_version("mlpstorage")
VERSION = _pkg_version("mlpstorage_py")
except _PkgNF:
VERSION = "unknown"
__version__ = VERSION

# boto3/botocore are banned — install the blocker immediately so any
# transitive import attempt is caught regardless of which module triggers it.
from mlpstorage.ban_boto3 import install as _ban_boto3
_ban_boto3()
from mlpstorage_py.ban_boto3 import install as _ban_boto3
_ban_boto3()
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
the BenchmarkRegistry for dynamic CLI construction.
"""

from mlpstorage.benchmarks.dlio import TrainingBenchmark, CheckpointingBenchmark
from mlpstorage.benchmarks.vectordbbench import VectorDBBenchmark
from mlpstorage.benchmarks.kvcache import KVCacheBenchmark
from mlpstorage.registry import BenchmarkRegistry
from mlpstorage.cli import (
from mlpstorage_py.benchmarks.dlio import TrainingBenchmark, CheckpointingBenchmark
from mlpstorage_py.benchmarks.vectordbbench import VectorDBBenchmark
from mlpstorage_py.benchmarks.kvcache import KVCacheBenchmark
from mlpstorage_py.registry import BenchmarkRegistry
from mlpstorage_py.cli import (
PROGRAM_DESCRIPTIONS,
add_training_arguments,
add_checkpointing_arguments,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,20 @@ def _run(self):

from pyarrow.ipc import open_stream

from mlpstorage.config import PARAM_VALIDATION, DATETIME_STR, MLPS_DEBUG, EXEC_TYPE
from mlpstorage.debug import debug_tryer_wrapper
from mlpstorage.interfaces import BenchmarkInterface, BenchmarkConfig, BenchmarkCommand
from mlpstorage.mlps_logging import setup_logging, apply_logging_options
from mlpstorage.rules import BenchmarkVerifier, generate_output_location, ClusterInformation
from mlpstorage.rules.models import ClusterSnapshots, TimeSeriesData, TimeSeriesSample
from mlpstorage.utils import CommandExecutor, MLPSJsonEncoder
from mlpstorage.cluster_collector import (
from mlpstorage_py.config import PARAM_VALIDATION, DATETIME_STR, MLPS_DEBUG, EXEC_TYPE
from mlpstorage_py.debug import debug_tryer_wrapper
from mlpstorage_py.interfaces import BenchmarkInterface, BenchmarkConfig, BenchmarkCommand
from mlpstorage_py.mlps_logging import setup_logging, apply_logging_options
from mlpstorage_py.rules import BenchmarkVerifier, generate_output_location, ClusterInformation
from mlpstorage_py.rules.models import ClusterSnapshots, TimeSeriesData, TimeSeriesSample
from mlpstorage_py.utils import CommandExecutor, MLPSJsonEncoder
from mlpstorage_py.cluster_collector import (
collect_cluster_info,
SSHClusterCollector,
TimeSeriesCollector,
MultiHostTimeSeriesCollector,
)
from mlpstorage.progress import create_stage_progress, progress_context
from mlpstorage_py.progress import create_stage_progress, progress_context

if TYPE_CHECKING:
import logging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
import pprint
import sys

from mlpstorage.benchmarks.base import Benchmark
from mlpstorage.config import (CONFIGS_ROOT_DIR, BENCHMARK_TYPES, EXEC_TYPE, MPIRUN, MLPSTORAGE_BIN_NAME,
from mlpstorage_py.benchmarks.base import Benchmark
from mlpstorage_py.config import (CONFIGS_ROOT_DIR, BENCHMARK_TYPES, EXEC_TYPE, MPIRUN, MLPSTORAGE_BIN_NAME,
LLM_ALLOWED_VALUES, LLM_SUBSET_PROCS, EXIT_CODE, MODELS, HYDRA_OUTPUT_SUBDIR,
LLM_SIZE_BY_RANK)
from mlpstorage.dependency_check import validate_benchmark_dependencies
from mlpstorage.rules import calculate_training_data_size, HostInfo, HostMemoryInfo, HostCPUInfo, ClusterInformation
from mlpstorage.utils import (read_config_from_file, create_nested_dict, update_nested_dict, generate_mpi_prefix_cmd)
from mlpstorage_py.dependency_check import validate_benchmark_dependencies
from mlpstorage_py.rules import calculate_training_data_size, HostInfo, HostMemoryInfo, HostCPUInfo, ClusterInformation
from mlpstorage_py.utils import (read_config_from_file, create_nested_dict, update_nested_dict, generate_mpi_prefix_cmd)


class DLIOBenchmark(Benchmark, abc.ABC):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
KV Cache Benchmark for MLPerf Storage.

This module provides the KVCacheBenchmark class that wraps the kv-cache.py
benchmark script for integration into the mlpstorage framework.
benchmark script for integration into the mlpstorage_py framework.

The KV Cache benchmark simulates storage system performance for Large Language
Model (LLM) Key-Value cache offloading, including:
Expand All @@ -20,17 +20,17 @@
from pathlib import Path
from typing import Dict, Any, Optional, List

from mlpstorage.benchmarks.base import Benchmark
from mlpstorage.config import (
from mlpstorage_py.benchmarks.base import Benchmark
from mlpstorage_py.config import (
BENCHMARK_TYPES,
EXEC_TYPE,
KVCACHE_MODELS,
KVCACHE_DEFAULT_DURATION,
KVCACHE_PERFORMANCE_PROFILES,
KVCACHE_GENERATION_MODES,
)
from mlpstorage.interfaces import BenchmarkCommand
from mlpstorage.utils import generate_mpi_prefix_cmd
from mlpstorage_py.interfaces import BenchmarkCommand
from mlpstorage_py.utils import generate_mpi_prefix_cmd


class KVCacheBenchmark(Benchmark):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
import sys
from typing import Dict, Any

from mlpstorage.benchmarks.base import Benchmark
from mlpstorage.config import CONFIGS_ROOT_DIR, BENCHMARK_TYPES
from mlpstorage.utils import read_config_from_file
from mlpstorage_py.benchmarks.base import Benchmark
from mlpstorage_py.config import CONFIGS_ROOT_DIR, BENCHMARK_TYPES
from mlpstorage_py.utils import read_config_from_file


class VectorDBBenchmark(Benchmark):
Expand Down
18 changes: 9 additions & 9 deletions mlpstorage/cli/__init__.py → mlpstorage_py/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
- utility_args: Reports and history arguments

Usage:
from mlpstorage.cli import add_training_arguments
from mlpstorage.registry import BenchmarkRegistry
from mlpstorage_py.cli import add_training_arguments
from mlpstorage_py.registry import BenchmarkRegistry

BenchmarkRegistry.register(
'training',
Expand All @@ -22,7 +22,7 @@
)
"""

from mlpstorage.cli.common_args import (
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
PROGRAM_DESCRIPTIONS,
add_universal_arguments,
Expand All @@ -31,12 +31,12 @@
add_dlio_arguments,
)

from mlpstorage.cli.training_args import add_training_arguments
from mlpstorage.cli.checkpointing_args import add_checkpointing_arguments
from mlpstorage.cli.vectordb_args import add_vectordb_arguments
from mlpstorage.cli.kvcache_args import add_kvcache_arguments
from mlpstorage.cli.utility_args import add_reports_arguments, add_history_arguments
from mlpstorage.cli.lockfile_args import add_lockfile_arguments
from mlpstorage_py.cli.training_args import add_training_arguments
from mlpstorage_py.cli.checkpointing_args import add_checkpointing_arguments
from mlpstorage_py.cli.vectordb_args import add_vectordb_arguments
from mlpstorage_py.cli.kvcache_args import add_kvcache_arguments
from mlpstorage_py.cli.utility_args import add_reports_arguments, add_history_arguments
from mlpstorage_py.cli.lockfile_args import add_lockfile_arguments

__all__ = [
# Common
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
including datasize and run commands.
"""

from mlpstorage.config import DEFAULT_HOSTS, EXEC_TYPE
from mlpstorage.cli.common_args import (
from mlpstorage_py.config import DEFAULT_HOSTS, EXEC_TYPE
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
add_universal_arguments,
add_mpi_arguments,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
- Validation utilities
"""

from mlpstorage.config import (
from mlpstorage_py.config import (
CHECKPOINT_RANKS_STRINGS, MODELS, ACCELERATORS, DEFAULT_HOSTS,
LLM_MODELS_STRINGS, MPI_CMDS, EXEC_TYPE, DEFAULT_RESULTS_DIR,
VECTOR_DTYPES, DISTRIBUTIONS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
including run and datasize commands for LLM inference storage testing.
"""

from mlpstorage.config import (
from mlpstorage_py.config import (
KVCACHE_MODELS,
KVCACHE_PERFORMANCE_PROFILES,
KVCACHE_GENERATION_MODES,
KVCACHE_DEFAULT_DURATION,
EXEC_TYPE,
)
from mlpstorage.cli.common_args import (
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
add_universal_arguments,
add_host_arguments,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
CLI argument builder for lockfile management commands.

Provides arguments for:
- mlpstorage lockfile generate: Create lockfile from pyproject.toml
- mlpstorage lockfile verify: Validate installed packages against lockfile
- mlpstorage_py lockfile generate: Create lockfile from pyproject.toml
- mlpstorage_py lockfile verify: Validate installed packages against lockfile
"""

from mlpstorage.cli.common_args import add_universal_arguments
from mlpstorage_py.cli.common_args import add_universal_arguments


def add_lockfile_arguments(parser):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
including datasize, datagen, run, and configview commands.
"""

from mlpstorage.config import MODELS, ACCELERATORS, DEFAULT_HOSTS, EXEC_TYPE
from mlpstorage.cli.common_args import (
from mlpstorage_py.config import MODELS, ACCELERATORS, DEFAULT_HOSTS, EXEC_TYPE
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
add_universal_arguments,
add_mpi_arguments,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
reports and history.
"""

from mlpstorage.cli.common_args import (
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
add_universal_arguments,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
including datagen and run commands.
"""

from mlpstorage.config import VECTOR_DTYPES, DISTRIBUTIONS, VECTORDB_DEFAULT_RUNTIME
from mlpstorage.cli.common_args import (
from mlpstorage_py.config import VECTOR_DTYPES, DISTRIBUTIONS, VECTORDB_DEFAULT_RUNTIME
from mlpstorage_py.cli.common_args import (
HELP_MESSAGES,
add_universal_arguments,
add_timeseries_arguments,
Expand Down
8 changes: 4 additions & 4 deletions mlpstorage/cli_parser.py → mlpstorage_py/cli_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
import argparse
import sys

from mlpstorage import VERSION
from mlpstorage.config import LLM_MODELS, VECTORDB_DEFAULT_RUNTIME, EXIT_CODE
from mlpstorage_py import VERSION
from mlpstorage_py.config import LLM_MODELS, VECTORDB_DEFAULT_RUNTIME, EXIT_CODE

# Import modular argument builders from cli package
from mlpstorage.cli import (
from mlpstorage_py.cli import (
HELP_MESSAGES,
PROGRAM_DESCRIPTIONS,
add_universal_arguments,
Expand All @@ -34,7 +34,7 @@ def parse_arguments():
Returns:
argparse.Namespace: Parsed and validated arguments.
"""
parser = argparse.ArgumentParser(description="Script to launch the MLPerf Storage benchmark")
parser = argparse.ArgumentParser(prog="mlpstorage", description="Script to launch the MLPerf Storage benchmark")
parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}")
sub_programs = parser.add_subparsers(dest="program", required=True)
sub_programs.required = True
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
from dataclasses import dataclass, field, asdict
from typing import Any, Dict, List, Optional, Tuple

from mlpstorage.config import MPIRUN, MPIEXEC, MPI_RUN_BIN, MPI_EXEC_BIN
from mlpstorage.interfaces.collector import ClusterCollectorInterface, CollectionResult
from mlpstorage_py.config import MPIRUN, MPIEXEC, MPI_RUN_BIN, MPI_EXEC_BIN
from mlpstorage_py.interfaces.collector import ClusterCollectorInterface, CollectionResult


# =============================================================================
Expand Down
2 changes: 1 addition & 1 deletion mlpstorage/config.py → mlpstorage_py/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,4 +175,4 @@ class BENCHMARK_TYPES(enum.Enum):
DISTRIBUTIONS = ["uniform", "normal", "zipfian"]

# Default runtime for vector database benchmarks if not defined
VECTORDB_DEFAULT_RUNTIME = 60
VECTORDB_DEFAULT_RUNTIME = 60
2 changes: 1 addition & 1 deletion mlpstorage/debug.py → mlpstorage_py/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from functools import partialmethod, wraps

from mlpstorage.config import MLPS_DEBUG
from mlpstorage_py.config import MLPS_DEBUG


def debug_tryer_wrapper(on_error, debug, logger, func):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import shutil
from typing import Optional, Tuple, List

from mlpstorage.environment import detect_os, get_install_instruction
from mlpstorage.error_messages import format_error
from mlpstorage.errors import DependencyError
from mlpstorage_py.environment import detect_os, get_install_instruction
from mlpstorage_py.error_messages import format_error
from mlpstorage_py.errors import DependencyError


def check_mpi_with_hints(mpi_bin: str = "mpirun") -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
collect_validation_issues: Function to separate errors from warnings
"""

from mlpstorage.environment.os_detect import OSInfo, detect_os
from mlpstorage.environment.install_hints import (
from mlpstorage_py.environment.os_detect import OSInfo, detect_os
from mlpstorage_py.environment.install_hints import (
get_install_instruction,
INSTALL_INSTRUCTIONS,
)
from mlpstorage.environment.validators import (
from mlpstorage_py.environment.validators import (
ValidationIssue,
validate_ssh_connectivity,
collect_validation_issues,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from typing import Optional

from mlpstorage.environment.os_detect import OSInfo
from mlpstorage_py.environment.os_detect import OSInfo


# Installation instructions keyed by (dependency, system, distro_id)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def validate_ssh_connectivity(
ssh_path = shutil.which('ssh')
if ssh_path is None:
# Import here to avoid circular imports
from mlpstorage.environment import detect_os, get_install_instruction
from mlpstorage_py.environment import detect_os, get_install_instruction

os_info = detect_os()
install_cmd = get_install_instruction('ssh', os_info)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
- Context-aware message generation

Usage:
from mlpstorage.error_messages import format_error, ERROR_MESSAGES
from mlpstorage_py.error_messages import format_error, ERROR_MESSAGES

# Format a known error
msg = format_error('CONFIG_MISSING_REQUIRED', param='model')
Expand Down
File renamed without changes.
Loading
Loading