feat: Adds configurable backoff to run_and_wait

.changelog/009.yaml

@@ -0,0 +1,92 @@
+ts: 2026-03-17 20:25:21.130919+00:00
+type: chore
+description: support new fields on ShellConfig to support retries
+---
+name: ''
+ts: 2026-03-17 20:25:49.365186+00:00
+type: fix
+author: Espen Albert
+ignored: true
+message: 'fix(test): Update ruff_ignore settings in test_examples to include I001'
+short_sha: 41cb07
+---
+name: ShellConfig
+ts: 2026-03-17 20:25:49.802449+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_field_added
+details: 'added optional field ''retry_initial_wait'' (default: 5)'
+field_name: retry_initial_wait
+group: shell
+---
+name: ShellConfig
+ts: 2026-03-17 20:25:49.802461+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_field_added
+details: 'added optional field ''retry_jitter'' (default: 5)'
+field_name: retry_jitter
+group: shell
+---
+name: ShellConfig
+ts: 2026-03-17 20:25:49.802464+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_field_added
+details: 'added optional field ''retry_max_wait'' (default: 60)'
+field_name: retry_max_wait
+group: shell
+---
+name: run
+ts: 2026-03-17 20:41:23.238962+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_initial_wait'' (default: None)'
+field_name: retry_initial_wait
+group: shell
+---
+name: run
+ts: 2026-03-17 20:41:23.238965+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_jitter'' (default: None)'
+field_name: retry_jitter
+group: shell
+---
+name: run
+ts: 2026-03-17 20:41:23.238968+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_max_wait'' (default: None)'
+field_name: retry_max_wait
+group: shell
+---
+name: run_and_wait
+ts: 2026-03-17 20:41:23.238970+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_initial_wait'' (default: None)'
+field_name: retry_initial_wait
+group: shell
+---
+name: run_and_wait
+ts: 2026-03-17 20:41:23.238972+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_jitter'' (default: None)'
+field_name: retry_jitter
+group: shell
+---
+name: run_and_wait
+ts: 2026-03-17 20:41:23.238975+00:00
+type: additional_change
+auto_generated: true
+change_kind: optional_param_added
+details: 'added optional param ''retry_max_wait'' (default: None)'
+field_name: retry_max_wait
+group: shell

ask_shell/_internal/_run.py

@@ -13,6 +13,7 @@
 import atexit
 import logging
 import os
+import random
 import signal
 import subprocess
 import sys
@@ -382,11 +383,20 @@ def _eval_should_retry(config: ShellConfig, result: ShellRun) -> bool:
         return False
 
 
+def _backoff_wait(attempt: int, config: ShellConfig) -> float:
+    """wait = min(initial * 2^(attempt-2), max_wait) + uniform(0, jitter)"""
+    base = min(config.retry_initial_wait * 2 ** (attempt - 2), config.retry_max_wait)
+    return base + random.uniform(0, config.retry_jitter)
+
+
 def _run_attempts(shell_run: ShellRun, output_dir: Path) -> BaseException | None:
     config = shell_run.config
     queue = shell_run._queue
     for attempt in range(1, config.attempts + 1):
         if attempt > 1:
+            wait_seconds = _backoff_wait(attempt, config)
+            logger.info(f"Backoff {wait_seconds:.2f}s before attempt {attempt}/{config.attempts}")
+            time.sleep(wait_seconds)
             queue.put_nowait(ShellRunRetryAttempt(attempt=attempt))
             logger.warning(f"Retrying run {shell_run} attempt {attempt} of {config.attempts}")
         attempt_log_prefix = config.run_log_stem(attempt)
@@ -459,6 +469,9 @@ def run(
     is_binary_call: bool | None = None,
     message_callbacks: list[Callable[[ShellRunEventT], bool]] | None = None,
     print_prefix: str | None = None,
+    retry_initial_wait: float | None = None,
+    retry_jitter: float | None = None,
+    retry_max_wait: float | None = None,
     run_log_stem_prefix: str | None = None,
     run_output_dir: Path | None | None = None,
     settings: AskShellSettings | None = None,
@@ -483,6 +496,9 @@ def run(
         is_binary_call=is_binary_call,
         message_callbacks=message_callbacks,
         print_prefix=print_prefix,
+        retry_initial_wait=retry_initial_wait,
+        retry_jitter=retry_jitter,
+        retry_max_wait=retry_max_wait,
         run_log_stem_prefix=run_log_stem_prefix,
         run_output_dir=run_output_dir,
         settings=settings,
@@ -518,6 +534,9 @@ def run_and_wait(
     is_binary_call: bool | None = None,
     message_callbacks: list[Callable[[ShellRunEventT], bool]] | None = None,
     print_prefix: str | None = None,
+    retry_initial_wait: float | None = None,
+    retry_jitter: float | None = None,
+    retry_max_wait: float | None = None,
     run_log_stem_prefix: str | None = None,
     run_output_dir: Path | None | None = None,
     settings: AskShellSettings | None = None,
@@ -542,6 +561,9 @@ def run_and_wait(
         is_binary_call=is_binary_call,
         message_callbacks=message_callbacks,
         print_prefix=print_prefix,
+        retry_initial_wait=retry_initial_wait,
+        retry_jitter=retry_jitter,
+        retry_max_wait=retry_max_wait,
         run_log_stem_prefix=run_log_stem_prefix,
         run_output_dir=run_output_dir,
         settings=settings,

ask_shell/_internal/models.py

@@ -151,6 +151,9 @@ class ShellConfig(Entity):
     # retry args
     attempts: int = 1
     should_retry: Callable[[ShellRun], bool] = always_retry
+    retry_initial_wait: float = Field(default=5, description="Seconds before first retry, only used when attempts > 1")
+    retry_max_wait: float = Field(default=60, description="Cap on wait between retries")
+    retry_jitter: float = Field(default=5, description="Max random jitter added to wait")
 
     # logging/output
     print_prefix: str = Field(default=None, description="Use cwd+binary_name+first_arg if not provided")  # type: ignore

ask_shell/public_api_test.py

@@ -5,6 +5,7 @@
 import time
 from json import loads
 from os import getenv
+from unittest.mock import patch
 
 import pytest
 from model_lib import fields
@@ -364,6 +365,51 @@ def test_mise_resolve(tmp_path):
     assert "not found" not in result.stdout_one_line
 
 
+def test_backoff_wait_values(tmp_path):
+    script_path = tmp_path / "attempt.py"
+    script_path.write_text(_attempt_script)
+    sleep_values: list[float] = []
+    original_sleep = time.sleep
+
+    def mock_sleep(seconds):
+        sleep_values.append(seconds)
+        original_sleep(0.01)
+
+    with patch.object(time, time.sleep.__name__, side_effect=mock_sleep):
+        result = run_and_wait(
+            f"{PYTHON_EXEC} {script_path}",
+            attempts=4,
+            retry_initial_wait=2,
+            retry_max_wait=10,
+            retry_jitter=0,
+        )
+    assert result.clean_complete
+    assert sleep_values == [2, 4]
+
+
+def test_backoff_respects_max_wait(tmp_path):
+    script_path = tmp_path / "fail.py"
+    script_path.write_text("raise SystemExit(1)\n")
+    sleep_values: list[float] = []
+    original_sleep = time.sleep
+
+    def mock_sleep(seconds):
+        sleep_values.append(seconds)
+        original_sleep(0.01)
+
+    with patch.object(time, time.sleep.__name__, side_effect=mock_sleep):
+        with pytest.raises(ShellError):
+            run_and_wait(
+                f"{PYTHON_EXEC} {script_path}",
+                attempts=5,
+                retry_initial_wait=2,
+                retry_max_wait=5,
+                retry_jitter=0,
+                skip_binary_check=True,
+            )
+    assert sleep_values == [2, 4, 5, 5]
+
+
 def test_readme_example():
     with question_patcher(responses=["y", "myname"]):
         assert ask.confirm("Run?")

ask_shell/test_docs.py

@@ -8,7 +8,7 @@
 
 @pytest.mark.parametrize("example", find_examples(EXAMPLES_DIR), ids=str)
 def test_examples(example: CodeExample, eval_example: EvalExample):
-    eval_example.set_config(line_length=120, target_version="py313", ruff_ignore=["T"])  # pyright: ignore[reportArgumentType]
+    eval_example.set_config(line_length=120, target_version="py313", ruff_ignore=["I001", "T"])  # pyright: ignore[reportArgumentType]
     prefix = example.prefix_settings()
     if prefix.get("test", "").startswith("skip"):
         pytest.skip(prefix["test"])

docs/examples/shell/AbortRetryError.md

@@ -12,7 +12,6 @@ The exception propagates as `ShellError.base_error`.
 import sys
 
 import pytest
-
 from ask_shell.shell import (
     AbortRetryError,
     ShellConfig,

docs/examples/shell/backoff.md

@@ -0,0 +1,47 @@
+<!--
+description: Configurable exponential backoff with jitter for retries
+-->
+# backoff
+
+`ShellConfig` supports exponential backoff between retry attempts via three fields:
+`retry_initial_wait`, `retry_max_wait`, and `retry_jitter`.
+
+When `attempts > 1`, the wait before retry N is `min(initial_wait * 2^(N-2), max_wait) + uniform(0, jitter)`.
+
+```python
+import sys
+import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ask_shell.shell import run_and_wait
+
+script = """\
+from pathlib import Path
+p = Path(__file__).with_name("attempt")
+n = int(p.read_text()) if p.exists() else 1
+p.write_text(str(n + 1))
+if n < 4:
+    raise SystemExit(1)
+print(f"ok on attempt {n}")
+"""
+
+with TemporaryDirectory() as tmp:
+    script_path = Path(tmp) / "retry.py"
+    script_path.write_text(script)
+    start = time.monotonic()
+    result = run_and_wait(
+        f"{sys.executable} {script_path}",
+        attempts=4,
+        retry_initial_wait=0.1,
+        retry_max_wait=10,
+        retry_jitter=0,
+    )
+    elapsed = time.monotonic() - start
+    print(result.stdout)
+    #> ok on attempt 4
+    print(f"waited at least 0.7s: {elapsed >= 0.7}")  # 0.1+0.2+0.4
+    #> waited at least 0.7s: True
+    print(f"waited less than 2s: {elapsed < 2}")
+    #> waited less than 2s: True
+```

docs/shell/index.md

@@ -27,7 +27,7 @@
 <a id="shellrun_def"></a>
 
 ### class: `ShellRun`
-- [source](../../ask_shell/_internal/models.py#L269)
+- [source](../../ask_shell/_internal/models.py#L272)
 > **Since:** 0.3.0
 
 ```python
@@ -58,7 +58,7 @@ Args:
 <a id="handle_interrupt_wait_def"></a>
 
 ### class: `handle_interrupt_wait`
-- [source](../../ask_shell/_internal/_run.py#L141)
+- [source](../../ask_shell/_internal/_run.py#L142)
 > **Since:** 0.3.0
 
 ```python
@@ -82,7 +82,7 @@ class handle_interrupt_wait:
 <a id="kill_def"></a>
 
 ### function: `kill`
-- [source](../../ask_shell/_internal/_run.py#L73)
+- [source](../../ask_shell/_internal/_run.py#L74)
 > **Since:** 0.3.0
 
 ```python
@@ -102,7 +102,7 @@ https://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess
 <a id="kill_all_runs_def"></a>
 
 ### function: `kill_all_runs`
-- [source](../../ask_shell/_internal/_run.py#L106)
+- [source](../../ask_shell/_internal/_run.py#L107)
 > **Since:** 0.3.0
 
 ```python
@@ -120,7 +120,7 @@ def kill_all_runs(immediate: bool = False, reason: str = '', abort_timeout: floa
 <a id="run_error_def"></a>
 
 ### function: `run_error`
-- [source](../../ask_shell/_internal/_run.py#L567)
+- [source](../../ask_shell/_internal/_run.py#L589)
 > **Since:** 0.3.0
 
 ```python
@@ -174,7 +174,7 @@ class run_pool:
 <a id="stop_runs_and_pool_def"></a>
 
 ### function: `stop_runs_and_pool`
-- [source](../../ask_shell/_internal/_run.py#L133)
+- [source](../../ask_shell/_internal/_run.py#L134)
 > **Since:** 0.3.0
 
 ```python
@@ -192,7 +192,7 @@ def stop_runs_and_pool(reason: str = 'atexit', immediate: bool = False):
 <a id="wait_on_ok_errors_def"></a>
 
 ### function: `wait_on_ok_errors`
-- [source](../../ask_shell/_internal/_run.py#L574)
+- [source](../../ask_shell/_internal/_run.py#L596)
 > **Since:** 0.3.0
 
 ```python
@@ -210,7 +210,7 @@ def wait_on_ok_errors(*runs, timeout: float | None = None, skip_kill_timeouts: b
 <a id="abortretryerror_def"></a>
 
 ### exception: `AbortRetryError`
-- [source](../../ask_shell/_internal/models.py#L520)
+- [source](../../ask_shell/_internal/models.py#L523)
 - [Example: Stop retrying with a custom error by raising AbortRetryError from should_retry](../examples/shell/AbortRetryError.md)
 > **Since:** 0.5.0
 

docs/shell/run.md

@@ -2,11 +2,11 @@
 
 <!-- === DO_NOT_EDIT: pkg-ext run_def === -->
 ## function: run
-- [source](../../ask_shell/_internal/_run.py#L450)
+- [source](../../ask_shell/_internal/_run.py#L460)
 > **Since:** 0.3.0
 
 ```python
-def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_html_log_files: bool | None = None, skip_progress_output: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, start_timeout: float | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun:
+def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, ansi_content: bool | None = None, attempts: int | None = None, cwd: str | Path | None = None, env: dict[str, str] | None = None, extra_popen_kwargs: dict | None = None, is_binary_call: bool | None = None, message_callbacks: list[Callable[[ShellRunBefore | ShellRunPOpenStarted | ShellRunStdStarted | ShellRunStdReadError | ShellRunStdOutput | ShellRunRetryAttempt | ShellRunAfter], bool]] | None = None, print_prefix: str | None = None, retry_initial_wait: float | None = None, retry_jitter: float | None = None, retry_max_wait: float | None = None, run_log_stem_prefix: str | None = None, run_output_dir: Path | None = None, settings: AskShellSettings | None = None, should_retry: Callable[[ShellRun], bool] | None = None, skip_binary_check: bool | None = None, skip_html_log_files: bool | None = None, skip_progress_output: bool | None = None, include_log_time: bool | None = None, skip_os_env: bool | None = None, start_timeout: float | None = None, terminal_width: int | None = None, skip_interactive_check: bool | None = None) -> ShellRun:
     ...
 ```
 <!-- === OK_EDIT: pkg-ext run_def === -->
@@ -16,6 +16,9 @@ def run(config: ShellConfig | str, *, allow_non_zero_exit: bool | None = None, a
 
 | Version | Change |
 |---------|--------|
+| unreleased | added optional param 'retry_max_wait' (default: None) |
+| unreleased | added optional param 'retry_jitter' (default: None) |
+| unreleased | added optional param 'retry_initial_wait' (default: None) |
 | 0.4.0 | param 'message_callbacks' type: list[Callable[[typing.Union[ask_shell._internal.events.ShellRunBefore, ask_shell._internal.events.ShellRunPOpenStarted, ask_shell._internal.events.ShellRunStdStarted, ask_shell._internal.events.ShellRunStdReadError, ask_shell._internal.events.ShellRunStdOutput, ask_shell._internal.events.ShellRunRetryAttempt, ask_shell._internal.events.ShellRunAfter]], bool]] | None -> list[Callable[[ask_shell._internal.events.ShellRunBefore | ask_shell._internal.events.ShellRunPOpenStarted | ask_shell._internal.events.ShellRunStdStarted | ask_shell._internal.events.ShellRunStdReadError | ask_shell._internal.events.ShellRunStdOutput | ask_shell._internal.events.ShellRunRetryAttempt | ask_shell._internal.events.ShellRunAfter], bool]] | None |
 | 0.3.0 | Made public |
 <!-- === OK_EDIT: pkg-ext run_changes === -->