diff --git a/.changelog/009.yaml b/.changelog/009.yaml new file mode 100644 index 0000000..a7c851b --- /dev/null +++ b/.changelog/009.yaml @@ -0,0 +1,92 @@ +ts: 2026-03-17 20:25:21.130919+00:00 +type: chore +description: support new fields on ShellConfig to support retries +--- +name: '' +ts: 2026-03-17 20:25:49.365186+00:00 +type: fix +author: Espen Albert +ignored: true +message: 'fix(test): Update ruff_ignore settings in test_examples to include I001' +short_sha: 41cb07 +--- +name: ShellConfig +ts: 2026-03-17 20:25:49.802449+00:00 +type: additional_change +auto_generated: true +change_kind: optional_field_added +details: 'added optional field ''retry_initial_wait'' (default: 5)' +field_name: retry_initial_wait +group: shell +--- +name: ShellConfig +ts: 2026-03-17 20:25:49.802461+00:00 +type: additional_change +auto_generated: true +change_kind: optional_field_added +details: 'added optional field ''retry_jitter'' (default: 5)' +field_name: retry_jitter +group: shell +--- +name: ShellConfig +ts: 2026-03-17 20:25:49.802464+00:00 +type: additional_change +auto_generated: true +change_kind: optional_field_added +details: 'added optional field ''retry_max_wait'' (default: 60)' +field_name: retry_max_wait +group: shell +--- +name: run +ts: 2026-03-17 20:41:23.238962+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_initial_wait'' (default: None)' +field_name: retry_initial_wait +group: shell +--- +name: run +ts: 2026-03-17 20:41:23.238965+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_jitter'' (default: None)' +field_name: retry_jitter +group: shell +--- +name: run +ts: 2026-03-17 20:41:23.238968+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_max_wait'' (default: None)' +field_name: retry_max_wait +group: shell +--- +name: run_and_wait +ts: 2026-03-17 20:41:23.238970+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_initial_wait'' (default: None)' +field_name: retry_initial_wait +group: shell +--- +name: run_and_wait +ts: 2026-03-17 20:41:23.238972+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_jitter'' (default: None)' +field_name: retry_jitter +group: shell +--- +name: run_and_wait +ts: 2026-03-17 20:41:23.238975+00:00 +type: additional_change +auto_generated: true +change_kind: optional_param_added +details: 'added optional param ''retry_max_wait'' (default: None)' +field_name: retry_max_wait +group: shell diff --git a/ask_shell/_internal/_run.py b/ask_shell/_internal/_run.py index 20d7891..c474c87 100644 --- a/ask_shell/_internal/_run.py +++ b/ask_shell/_internal/_run.py @@ -13,6 +13,7 @@ import atexit import logging import os +import random import signal import subprocess import sys @@ -382,11 +383,20 @@ def _eval_should_retry(config: ShellConfig, result: ShellRun) -> bool: return False +def _backoff_wait(attempt: int, config: ShellConfig) -> float: + """wait = min(initial * 2^(attempt-2), max_wait) + uniform(0, jitter)""" + base = min(config.retry_initial_wait * 2 ** (attempt - 2), config.retry_max_wait) + return base + random.uniform(0, config.retry_jitter) + + def _run_attempts(shell_run: ShellRun, output_dir: Path) -> BaseException | None: config = shell_run.config queue = shell_run._queue for attempt in range(1, config.attempts + 1): if attempt > 1: + wait_seconds = _backoff_wait(attempt, config) + logger.info(f"Backoff {wait_seconds:.2f}s before attempt {attempt}/{config.attempts}") + time.sleep(wait_seconds) queue.put_nowait(ShellRunRetryAttempt(attempt=attempt)) logger.warning(f"Retrying run {shell_run} attempt {attempt} of {config.attempts}") attempt_log_prefix = config.run_log_stem(attempt) @@ -459,6 +469,9 @@ def run( is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunEventT], bool]] | None = None, print_prefix: str | None = None, + retry_initial_wait: float | None = None, + retry_jitter: float | None = None, + retry_max_wait: float | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None | None = None, settings: AskShellSettings | None = None, @@ -483,6 +496,9 @@ def run( is_binary_call=is_binary_call, message_callbacks=message_callbacks, print_prefix=print_prefix, + retry_initial_wait=retry_initial_wait, + retry_jitter=retry_jitter, + retry_max_wait=retry_max_wait, run_log_stem_prefix=run_log_stem_prefix, run_output_dir=run_output_dir, settings=settings, @@ -518,6 +534,9 @@ def run_and_wait( is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunEventT], bool]] | None = None, print_prefix: str | None = None, + retry_initial_wait: float | None = None, + retry_jitter: float | None = None, + retry_max_wait: float | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None | None = None, settings: AskShellSettings | None = None, @@ -542,6 +561,9 @@ def run_and_wait( is_binary_call=is_binary_call, message_callbacks=message_callbacks, print_prefix=print_prefix, + retry_initial_wait=retry_initial_wait, + retry_jitter=retry_jitter, + retry_max_wait=retry_max_wait, run_log_stem_prefix=run_log_stem_prefix, run_output_dir=run_output_dir, settings=settings, diff --git a/ask_shell/_internal/models.py b/ask_shell/_internal/models.py index 008d18d..1889a07 100644 --- a/ask_shell/_internal/models.py +++ b/ask_shell/_internal/models.py @@ -151,6 +151,9 @@ class ShellConfig(Entity): # retry args attempts: int = 1 should_retry: Callable[[ShellRun], bool] = always_retry + retry_initial_wait: float = Field(default=5, description="Seconds before first retry, only used when attempts > 1") + retry_max_wait: float = Field(default=60, description="Cap on wait between retries") + retry_jitter: float = Field(default=5, description="Max random jitter added to wait") # logging/output print_prefix: str = Field(default=None, description="Use cwd+binary_name+first_arg if not provided") # type: ignore diff --git a/ask_shell/public_api_test.py b/ask_shell/public_api_test.py index 5086d37..caae7f6 100644 --- a/ask_shell/public_api_test.py +++ b/ask_shell/public_api_test.py @@ -5,6 +5,7 @@ import time from json import loads from os import getenv +from unittest.mock import patch import pytest from model_lib import fields @@ -364,6 +365,51 @@ def test_mise_resolve(tmp_path): assert "not found" not in result.stdout_one_line +def test_backoff_wait_values(tmp_path): + script_path = tmp_path / "attempt.py" + script_path.write_text(_attempt_script) + sleep_values: list[float] = [] + original_sleep = time.sleep + + def mock_sleep(seconds): + sleep_values.append(seconds) + original_sleep(0.01) + + with patch.object(time, time.sleep.__name__, side_effect=mock_sleep): + result = run_and_wait( + f"{PYTHON_EXEC} {script_path}", + attempts=4, + retry_initial_wait=2, + retry_max_wait=10, + retry_jitter=0, + ) + assert result.clean_complete + assert sleep_values == [2, 4] + + +def test_backoff_respects_max_wait(tmp_path): + script_path = tmp_path / "fail.py" + script_path.write_text("raise SystemExit(1)\n") + sleep_values: list[float] = [] + original_sleep = time.sleep + + def mock_sleep(seconds): + sleep_values.append(seconds) + original_sleep(0.01) + + with patch.object(time, time.sleep.__name__, side_effect=mock_sleep): + with pytest.raises(ShellError): + run_and_wait( + f"{PYTHON_EXEC} {script_path}", + attempts=5, + retry_initial_wait=2, + retry_max_wait=5, + retry_jitter=0, + skip_binary_check=True, + ) + assert sleep_values == [2, 4, 5, 5] + + def test_readme_example(): with question_patcher(responses=["y", "myname"]): assert ask.confirm("Run?") diff --git a/ask_shell/test_docs.py b/ask_shell/test_docs.py index c2c3bb6..b919971 100644 --- a/ask_shell/test_docs.py +++ b/ask_shell/test_docs.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize("example", find_examples(EXAMPLES_DIR), ids=str) def test_examples(example: CodeExample, eval_example: EvalExample): - eval_example.set_config(line_length=120, target_version="py313", ruff_ignore=["T"]) # pyright: ignore[reportArgumentType] + eval_example.set_config(line_length=120, target_version="py313", ruff_ignore=["I001", "T"]) # pyright: ignore[reportArgumentType] prefix = example.prefix_settings() if prefix.get("test", "").startswith("skip"): pytest.skip(prefix["test"]) diff --git a/docs/examples/shell/AbortRetryError.md b/docs/examples/shell/AbortRetryError.md index 9f363df..fe20534 100644 --- a/docs/examples/shell/AbortRetryError.md +++ b/docs/examples/shell/AbortRetryError.md @@ -12,7 +12,6 @@ The exception propagates as `ShellError.base_error`. import sys import pytest - from ask_shell.shell import ( AbortRetryError, ShellConfig, diff --git a/docs/examples/shell/backoff.md b/docs/examples/shell/backoff.md new file mode 100644 index 0000000..320a6c5 --- /dev/null +++ b/docs/examples/shell/backoff.md @@ -0,0 +1,47 @@ + +# backoff + +`ShellConfig` supports exponential backoff between retry attempts via three fields: +`retry_initial_wait`, `retry_max_wait`, and `retry_jitter`. + +When `attempts > 1`, the wait before retry N is `min(initial_wait * 2^(N-2), max_wait) + uniform(0, jitter)`. + +```python +import sys +import time +from pathlib import Path +from tempfile import TemporaryDirectory + +from ask_shell.shell import run_and_wait + +script = """\ +from pathlib import Path +p = Path(__file__).with_name("attempt") +n = int(p.read_text()) if p.exists() else 1 +p.write_text(str(n + 1)) +if n < 4: + raise SystemExit(1) +print(f"ok on attempt {n}") +""" + +with TemporaryDirectory() as tmp: + script_path = Path(tmp) / "retry.py" + script_path.write_text(script) + start = time.monotonic() + result = run_and_wait( + f"{sys.executable} {script_path}", + attempts=4, + retry_initial_wait=0.1, + retry_max_wait=10, + retry_jitter=0, + ) + elapsed = time.monotonic() - start + print(result.stdout) + #> ok on attempt 4 + print(f"waited at least 0.7s: {elapsed >= 0.7}") # 0.1+0.2+0.4 + #> waited at least 0.7s: True + print(f"waited less than 2s: {elapsed < 2}") + #> waited less than 2s: True +``` diff --git a/docs/shell/index.md b/docs/shell/index.md index 62083c6..55d474d 100644 --- a/docs/shell/index.md +++ b/docs/shell/index.md @@ -27,7 +27,7 @@ ### class: `ShellRun` -- [source](../../ask_shell/_internal/models.py#L269) +- [source](../../ask_shell/_internal/models.py#L272) > **Since:** 0.3.0 ```python @@ -58,7 +58,7 @@ Args: ### class: `handle_interrupt_wait` -- [source](../../ask_shell/_internal/_run.py#L141) +- [source](../../ask_shell/_internal/_run.py#L142) > **Since:** 0.3.0 ```python @@ -82,7 +82,7 @@ class handle_interrupt_wait: ### function: `kill` -- [source](../../ask_shell/_internal/_run.py#L73) +- [source](../../ask_shell/_internal/_run.py#L74) > **Since:** 0.3.0 ```python @@ -102,7 +102,7 @@ https://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess ### function: `kill_all_runs` -- [source](../../ask_shell/_internal/_run.py#L106) +- [source](../../ask_shell/_internal/_run.py#L107) > **Since:** 0.3.0 ```python @@ -120,7 +120,7 @@ def kill_all_runs(immediate: bool = False, reason: str = '', abort_timeout: floa ### function: `run_error` -- [source](../../ask_shell/_internal/_run.py#L567) +- [source](../../ask_shell/_internal/_run.py#L589) > **Since:** 0.3.0 ```python @@ -174,7 +174,7 @@ class run_pool: ### function: `stop_runs_and_pool` -- [source](../../ask_shell/_internal/_run.py#L133) +- [source](../../ask_shell/_internal/_run.py#L134) > **Since:** 0.3.0 ```python @@ -192,7 +192,7 @@ def stop_runs_and_pool(reason: str = 'atexit', immediate: bool = False): ### function: `wait_on_ok_errors` -- [source](../../ask_shell/_internal/_run.py#L574) +- [source](../../ask_shell/_internal/_run.py#L596) > **Since:** 0.3.0 ```python @@ -210,7 +210,7 @@ def wait_on_ok_errors(*runs, timeout: float | None = None, skip_kill_timeouts: b ### exception: `AbortRetryError` -- [source](../../ask_shell/_internal/models.py#L520) +- [source](../../ask_shell/_internal/models.py#L523) - [Example: Stop retrying with a custom error by raising AbortRetryError from should_retry](../examples/shell/AbortRetryError.md) > **Since:** 0.5.0 diff --git a/docs/shell/run.md b/docs/shell/run.md index 24b2698..30ccbe1 100644 --- a/docs/shell/run.md +++ b/docs/shell/run.md @@ -2,11 +2,11 @@ ## function: run -- [source](../../ask_shell/_internal/_run.py#L450) +- [source](../../ask_shell/_internal/_run.py#L460) > **Since:** 0.3.0 ```python -def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_html_log_files: bool | None = None, skip_progress_output: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, start_timeout: float | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun: +def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, retry_initial_wait: float | None = None, retry_jitter: float | None = None, retry_max_wait: float | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_html_log_files: bool | None = None, skip_progress_output: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, start_timeout: float | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun: ... ``` @@ -16,6 +16,9 @@ def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, a | Version | Change | |---------|--------| +| unreleased | added optional param 'retry_max_wait' (default: None) | +| unreleased | added optional param 'retry_jitter' (default: None) | +| unreleased | added optional param 'retry_initial_wait' (default: None) | | 0.4.0 | param 'message_callbacks' type: list[Callable[[typing.Union[ask_shell._internal.events.ShellRunBefore, ask_shell._internal.events.ShellRunPOpenStarted, ask_shell._internal.events.ShellRunStdStarted, ask_shell._internal.events.ShellRunStdReadError, ask_shell._internal.events.ShellRunStdOutput, ask_shell._internal.events.ShellRunRetryAttempt, ask_shell._internal.events.ShellRunAfter]], bool]] | None -> list[Callable[[ask_shell._internal.events.ShellRunBefore | ask_shell._internal.events.ShellRunPOpenStarted | ask_shell._internal.events.ShellRunStdStarted | ask_shell._internal.events.ShellRunStdReadError | ask_shell._internal.events.ShellRunStdOutput | ask_shell._internal.events.ShellRunRetryAttempt | ask_shell._internal.events.ShellRunAfter], bool]] | None | | 0.3.0 | Made public | \ No newline at end of file diff --git a/docs/shell/run_and_wait.md b/docs/shell/run_and_wait.md index 25fe8b6..cebc6c2 100644 --- a/docs/shell/run_and_wait.md +++ b/docs/shell/run_and_wait.md @@ -2,11 +2,11 @@ ## function: run_and_wait -- [source](../../ask_shell/_internal/_run.py#L508) +- [source](../../ask_shell/_internal/_run.py#L524) > **Since:** 0.3.0 ```python -def run_and_wait(script: ShellConfig | str, timeout: float | None = None, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_progress_output: bool | None = None, skip_html_log_files: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, user_input: bool | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun: +def run_and_wait(script: ShellConfig | str, timeout: float | None = None, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, retry_initial_wait: float | None = None, retry_jitter: float | None = None, retry_max_wait: float | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_progress_output: bool | None = None, skip_html_log_files: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, user_input: bool | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun: ... ``` @@ -16,6 +16,9 @@ def run_and_wait(script: ShellConfig | str, timeout: float | None = None, *, all | Version | Change | |---------|--------| +| unreleased | added optional param 'retry_max_wait' (default: None) | +| unreleased | added optional param 'retry_jitter' (default: None) | +| unreleased | added optional param 'retry_initial_wait' (default: None) | | 0.4.0 | param 'message_callbacks' type: list[Callable[[typing.Union[ask_shell._internal.events.ShellRunBefore, ask_shell._internal.events.ShellRunPOpenStarted, ask_shell._internal.events.ShellRunStdStarted, ask_shell._internal.events.ShellRunStdReadError, ask_shell._internal.events.ShellRunStdOutput, ask_shell._internal.events.ShellRunRetryAttempt, ask_shell._internal.events.ShellRunAfter]], bool]] | None -> list[Callable[[ask_shell._internal.events.ShellRunBefore | ask_shell._internal.events.ShellRunPOpenStarted | ask_shell._internal.events.ShellRunStdStarted | ask_shell._internal.events.ShellRunStdReadError | ask_shell._internal.events.ShellRunStdOutput | ask_shell._internal.events.ShellRunRetryAttempt | ask_shell._internal.events.ShellRunAfter], bool]] | None | | 0.3.0 | Made public | \ No newline at end of file diff --git a/docs/shell/shellconfig.md b/docs/shell/shellconfig.md index 2bed24d..14fbf60 100644 --- a/docs/shell/shellconfig.md +++ b/docs/shell/shellconfig.md @@ -18,6 +18,9 @@ class ShellConfig(Entity): user_input: bool = False attempts: int = 1 should_retry: Callable[[ShellRun], bool] = + retry_initial_wait: float = 5 + retry_max_wait: float = 60 + retry_jitter: float = 5 print_prefix: str = None include_log_time: bool = False ansi_content: bool = None @@ -75,6 +78,9 @@ class ShellConfig(Entity): | Version | Change | |---------|--------| +| unreleased | added optional field 'retry_max_wait' (default: 60) | +| unreleased | added optional field 'retry_jitter' (default: 5) | +| unreleased | added optional field 'retry_initial_wait' (default: 5) | | 0.4.0 | field 'message_callbacks' type: list[Callable[[typing.Union[ask_shell._internal.events.ShellRunBefore, ask_shell._internal.events.ShellRunPOpenStarted, ask_shell._internal.events.ShellRunStdStarted, ask_shell._internal.events.ShellRunStdReadError, ask_shell._internal.events.ShellRunStdOutput, ask_shell._internal.events.ShellRunRetryAttempt, ask_shell._internal.events.ShellRunAfter]], bool | None]] -> list[Callable[[ask_shell._internal.events.ShellRunBefore | ask_shell._internal.events.ShellRunPOpenStarted | ask_shell._internal.events.ShellRunStdStarted | ask_shell._internal.events.ShellRunStdReadError | ask_shell._internal.events.ShellRunStdOutput | ask_shell._internal.events.ShellRunRetryAttempt | ask_shell._internal.events.ShellRunAfter], bool | None]] | | 0.3.2 | field 'print_prefix' default added: None | | 0.3.2 | field 'is_binary_call' default added: None | diff --git a/docs/shell/shellerror.md b/docs/shell/shellerror.md index efe496a..c90a4c5 100644 --- a/docs/shell/shellerror.md +++ b/docs/shell/shellerror.md @@ -2,7 +2,7 @@ ## exception: ShellError -- [source](../../ask_shell/_internal/models.py#L484) +- [source](../../ask_shell/_internal/models.py#L487) > **Since:** 0.3.0 ```python