diff --git a/.github/workflows/provable.yml b/.github/workflows/provable.yml new file mode 100644 index 0000000..ffc1b46 --- /dev/null +++ b/.github/workflows/provable.yml @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: MPL-2.0 +# Provable — machine-checks the claims Chapeliser makes outside the Rust layer. +# +# The Rust CLI/codegen is covered by rust-ci.yml. This workflow verifies the +# four things that are otherwise only asserted in prose: +# 1. idris2-proofs — the Idris2 ABI proofs actually type-check (`idris2 --check`). +# 2. zig-ffi — the Zig FFI reference implementation builds and its tests pass. +# 3. codegen-drift — the committed golden sample matches fresh codegen output. +# 4. chapel-golden — the generated Chapel actually compiles AND runs via `chpl`. +# +# Until every job here is green, the proofs / FFI / Chapel output are "written" +# but NOT "verified" — ROADMAP.adoc and README.adoc are worded accordingly. +name: Provable + +on: + push: + branches: [main, master, "claude/**"] + pull_request: + +permissions: + contents: read + +jobs: + idris2-proofs: + name: Idris2 — machine-check ABI proofs + runs-on: ubuntu-latest + timeout-minutes: 20 + container: ghcr.io/stefan-hoeck/idris2-pack:latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - name: idris2 --check the ABI modules + working-directory: src/interface/abi + # Modules are namespaced Chapeliser.ABI.*, so they live under + # Chapeliser/ABI/ (idris2 requires file path to match module name). + # Foreign imports Types + Layout, so checking it checks the lot; we + # check each explicitly for clearer failure attribution. + run: | + idris2 --version + idris2 --check Chapeliser/ABI/Types.idr + idris2 --check Chapeliser/ABI/Layout.idr + idris2 --check Chapeliser/ABI/Foreign.idr + + zig-ffi: + name: Zig — build + test FFI reference impl + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: mlugg/setup-zig@v1 # SHA-pin pending (advisory Hypatia medium) + with: + version: 0.14.0 + - name: zig build + test + working-directory: src/interface/ffi + run: | + zig version + zig build test + zig build + + codegen-drift: + name: Codegen — golden sample is up to date + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - name: Regenerate golden artifacts and diff against committed tree + run: | + cargo run --quiet -- generate -m examples/golden/echo.toml -o /tmp/golden + diff -ru examples/golden/generated /tmp/golden + + chapel-golden: + name: Chapel — compile + run golden sample + runs-on: ubuntu-latest + timeout-minutes: 20 + container: docker://chapel/chapel:latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - name: Compile generated Chapel against the echo FFI stub + run: | + chpl --version + chpl --main-module echo_Distributed \ + examples/golden/generated/chapel/echo_distributed.chpl \ + examples/golden/ffi_stub.c \ + -o /tmp/echo_distributed + - name: Run on a single locale and assert it conserves all items + run: | + /tmp/echo_distributed -nl1 | tee /tmp/echo.out + grep -q "Gathered 8/8 results (merge)" /tmp/echo.out + grep -q "complete in" /tmp/echo.out diff --git a/.machine_readable/6a2/STATE.a2ml b/.machine_readable/6a2/STATE.a2ml index 60cec69..d74f904 100644 --- a/.machine_readable/6a2/STATE.a2ml +++ b/.machine_readable/6a2/STATE.a2ml @@ -5,7 +5,7 @@ [metadata] project = "chapeliser" version = "0.1.0" -last-updated = "2026-03-21" +last-updated = "2026-06-18" status = "active" session = "converted from scheme — 2026-04-11" @@ -15,7 +15,7 @@ purpose = """General-purpose Chapel acceleration framework""" completion-percentage = 45 [position] -phase = "phase-1-complete" # design | implementation | testing | maintenance | archived +phase = "testing" # design | implementation | testing | maintenance | archived maturity = "experimental" # experimental | alpha | beta | production | lts [route-to-mvp] @@ -25,16 +25,17 @@ milestones = [ [blockers-and-issues] issues = [ - "Chapel compiler not installed on dev machine — generated code verified by structure, not compilation", + "Idris2/Zig/Chapel toolchains unavailable in the local dev sandbox — proofs, FFI, and generated Chapel are now machine-checked in CI (.github/workflows/provable.yml) rather than locally.", + "provable.yml has not yet had a green run — until it does, the Idris2 proofs, Zig FFI build, and golden Chapel compile/run are written but not confirmed (ROADMAP Phase 1b).", ] [critical-next-actions] actions = [ - "End-to-end test: generate + compile + run with panic-attacker workload", - "Add shell completions (bash, zsh, fish)", - "Write additional examples (Monte Carlo, image processing)", + "Drive .github/workflows/provable.yml to green (idris2-proofs, zig-ffi, codegen-drift, chapel-golden).", + "Add golden samples for the remaining partition/gather strategies and multi-locale runs.", + "Add shell completions (bash, zsh, fish).", ] [maintenance-status] -last-run-utc = "2026-03-21T00:00:00Z" -last-result = "unknown" # unknown | pass | warn | fail +last-run-utc = "2026-06-18T00:00:00Z" +last-result = "warn" # unknown | pass | warn | fail — Rust layer green (63 tests); provable.yml pending first run diff --git a/README.adoc b/README.adoc index 1175893..d3164a5 100644 --- a/README.adoc +++ b/README.adoc @@ -110,13 +110,15 @@ You write **zero Chapel code**. Chapeliser generates everything. Chapeliser follows the hyperpolymath ABI-FFI standard: -* **Idris2 ABI** (`src/abi/`) — Formal proofs that: - - Data layouts are consistent across nodes +* **Idris2 ABI** (`src/interface/abi/`) — Dependent-type *proof obligations*, + machine-checked in CI (`.github/workflows/provable.yml`); the Rust mirror in + `src/abi/` carries the matching runtime types and checks: - Partition functions produce complete, non-overlapping splits - Gather functions preserve all results - Serialization round-trips are identity + - Memory layouts are consistent across the FFI boundary -* **Zig FFI** (`ffi/zig/`) — C-ABI bridge between: +* **Zig FFI** (`src/interface/ffi/`) — C-ABI bridge between: - The user's application (any language with C FFI) - The Chapel runtime (`chpl_*` functions) - Memory management across the boundary @@ -138,7 +140,7 @@ Chapeliser follows the hyperpolymath ABI-FFI standard: [source,bash] ---- -# Install +# Install (once published to crates.io — see ROADMAP Phase 3) cargo install chapeliser # In your project directory, create chapeliser.toml (see above) @@ -158,7 +160,10 @@ hundreds of repositories on a compute cluster. == Status -**Pre-alpha.** Architecture defined, ABI proofs in progress, codegen planned. +**Pre-alpha.** The Rust CLI and code generator are implemented and tested +(63 tests). The Idris2 proofs, Zig FFI, and a golden Chapel compile-and-run are +wired into CI as the verification gate (`.github/workflows/provable.yml`) — see +`ROADMAP.adoc` Phase 1b for their live (green/red) status. Not yet released. == License diff --git a/ROADMAP.adoc b/ROADMAP.adoc index d223b42..767f3e8 100644 --- a/ROADMAP.adoc +++ b/ROADMAP.adoc @@ -12,8 +12,13 @@ Jonathan D.A. Jewell * [x] Example manifest (panic-attacker mass-panic) * [x] README with architecture, SECURITY, CONTRIBUTING, GOVERNANCE -== Phase 1: Core Implementation (COMPLETE) -* [x] Chapel code generator — produces compilable .chpl with real distribution logic +== Phase 1: Core Implementation (IMPLEMENTED — machine-verification gated in CI) + +NOTE: "Implemented" = the code is written and the Rust layer is tested. The +Idris2 proofs, Zig FFI, and generated Chapel are checked by `idris2`/`zig`/`chpl` +only in CI (`.github/workflows/provable.yml`); their live status is Phase 1b. + +* [x] Chapel code generator — produces .chpl with real distribution logic ** Per-item: even coforall distribution with locale range helper ** Chunk: fixed-size chunks, round-robin across locales ** Adaptive: work-stealing via Chapel DynamicIters @@ -31,28 +36,39 @@ Jonathan D.A. Jewell * [x] Zig FFI bridge generator — 12 Chapel-facing exports delegating to user code * [x] C header generator — full FFI contract with documentation * [x] Build script generator — env var overrides, Chapel config passthrough -* [x] Idris2 ABI proofs (Types.idr, Layout.idr, Foreign.idr) +* [x] Idris2 ABI proofs *written* (Types.idr, Layout.idr, Foreign.idr) — machine-checked in Phase 1b ** Partition completeness + disjointness ** Gather conservation ** Serialisation round-trip witness ** Retry isolation ** Memory layout proofs for item/result buffers -* [x] Zig FFI reference implementation (echo processor, FNV-1a hash, concatenation reducer) +* [x] Zig FFI reference implementation *written* (echo processor, FNV-1a hash, concatenation reducer) — compiled in Phase 1b * [x] Rust ABI types with runtime verification ** Partition, GatherResult, MemoryBudget, FfiResult ** Strategy enums with parsing - ** 6 unit tests + ** 22 unit tests * [x] Cluster config parsing (cluster.toml → GASNET/SSH env vars) * [x] Build command: release/debug mode passthrough -* [x] 15 tests passing (6 unit + 8 integration + 1 doc-test) +* [x] 63 tests passing in the Rust layer (22 unit + 40 integration + 1 doc-test) + +== Phase 1b: Machine-verification (gated in CI — `.github/workflows/provable.yml`) + +These convert the Phase 1 artifacts from "written" to "verified". Each item +turns `[x]` only once its CI job is green; until then the Idris2/Zig/Chapel +artifacts above are implemented but NOT yet machine-checked. + +* [ ] `idris2 --check` passes for Types.idr, Layout.idr, Foreign.idr (job: `idris2-proofs`) +* [ ] `zig build test` passes for the FFI reference impl (job: `zig-ffi`) +* [ ] Committed golden codegen output is drift-free (job: `codegen-drift`) +* [ ] Generated Chapel compiles + runs via `chpl` — echo sample, 8/8 items conserved (job: `chapel-golden`) == Phase 2: Polish * [ ] Error messages and diagnostics (human-readable codegen errors) * [ ] Shell completions (bash, zsh, fish) * [ ] `chapeliser info` — show memory budget estimate from manifest * [ ] Performance benchmarks (single-locale vs multi-locale scaling) -* [ ] Additional examples (Monte Carlo, image processing, text search) -* [ ] CI/CD for the generated Chapel artifacts +* [ ] Additional golden samples (other 4 partition × 5 gather strategies; multi-locale) +* [~] CI/CD for the generated Chapel artifacts — wired in `provable.yml` (see Phase 1b) == Phase 3: Ecosystem * [ ] PanLL panel integration (distribution visualisation) diff --git a/examples/golden/README.adoc b/examples/golden/README.adoc new file mode 100644 index 0000000..4c60a8c --- /dev/null +++ b/examples/golden/README.adoc @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MPL-2.0 += Golden sample — end-to-end verification fixture +Jonathan D.A. Jewell +:icons: font + +This directory is the *provable-real* fixture for Chapeliser: the one workload +that CI takes all the way from manifest to a running Chapel binary. + +== Why it exists + +Most of Chapeliser is exercised by Rust unit/integration tests. But the whole +point of the tool — _generate Chapel, compile it, run it_ — can only be proven +by actually invoking `chpl`. This fixture closes that loop in CI +(`.github/workflows/provable.yml`, job `chapel-golden`). + +== Contents + +[cols="1,3"] +|=== +| `echo.toml` | A deliberately minimal manifest: `per-item` partition, +`merge` gather. These are the simplest strategies, so the generated Chapel +pulls in *no* optional modules (no `BlockDist`, `DynamicIters`, or +`AtomicObjects`) — keeping the compile surface small and stable. + +| `generated/` | The Chapel wrapper, Zig FFI bridge, C header and build script +produced by `chapeliser generate -m echo.toml`. Committed so the output is +reviewable and drift-checked. CI regenerates and `diff`s against this tree +(job `codegen-drift`) — if they differ, the build fails. + +| `ffi_stub.c` | A ~60-line C implementation of the 12 `c_*` FFI functions for +a trivial echo workload (8 items, copy input to output). Stands in for real +user code so the generated Chapel can be linked and run. +|=== + +== Reproduce locally + +[source,bash] +---- +# 1. (Re)generate — must match the committed generated/ tree: +cargo run -- generate -m examples/golden/echo.toml -o examples/golden/generated + +# 2. Compile the generated Chapel against the echo FFI stub (needs chpl + cc): +chpl --main-module echo_Distributed \ + examples/golden/generated/chapel/echo_distributed.chpl \ + examples/golden/ffi_stub.c \ + -o /tmp/echo_distributed + +# 3. Run on a single locale — expect "Gathered 8/8 results (merge)": +/tmp/echo_distributed -nl1 +---- + +== What this proves + +* The Rust codegen emits Chapel that a real `chpl` accepts (not just + "structurally plausible" Chapel). +* The C-ABI contract between the generated Chapel `extern proc` declarations + and the FFI layer links cleanly. +* The per-item / merge distribution path runs to completion and conserves all + items (8 in, 8 gathered). + +It does *not* prove multi-locale (`-nl>1`) behaviour or the other four +partition/gather strategies; those are future fixtures (see `ROADMAP.adoc`). diff --git a/examples/golden/echo.toml b/examples/golden/echo.toml new file mode 100644 index 0000000..3739deb --- /dev/null +++ b/examples/golden/echo.toml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: MPL-2.0 +# Golden sample — a trivial "echo" workload whose only purpose is to let CI +# machine-verify the FULL Chapeliser pipeline end to end: +# codegen (Rust) -> Chapel compile (chpl) -> link FFI stub -> run. +# +# It deliberately uses the simplest strategies (per-item / merge) so the +# generated Chapel exercises the core distribution path WITHOUT pulling in +# optional modules (BlockDist, DynamicIters, AtomicObjects). Keeping the +# compile surface minimal makes this a stable verification fixture. +# +# Verified by: .github/workflows/provable.yml (chapel-golden job). + +[workload] +name = "echo" +entry = "examples/golden/echo.rs::echo" +partition = "per-item" +gather = "merge" + +[data] +input-type = "Vec" +output-type = "Vec" +serialization = "raw" +max-item-bytes = 4096 + +[scaling] +min-nodes = 1 +max-nodes = 4 +grain-size = 1 +expected-items = 8 diff --git a/examples/golden/ffi_stub.c b/examples/golden/ffi_stub.c new file mode 100644 index 0000000..719e836 --- /dev/null +++ b/examples/golden/ffi_stub.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: MPL-2.0 + * Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + * + * Golden-sample FFI stub — a minimal C implementation of the 12 Chapeliser + * C-ABI functions that the generated Chapel wrapper calls via `extern proc`. + * + * Its only purpose is to let CI compile AND RUN the generated Chapel end to + * end (see .github/workflows/provable.yml, chapel-golden job). It implements a + * trivial "echo" workload: 8 fixed items, each processed by copying input to + * output unchanged. This stands in for real user code so the generated + * distribution/gather/store logic can be exercised by `chpl`. + * + * Symbol names here match the `extern proc c_*` declarations emitted by + * src/codegen/chapel.rs. (In production the generated Zig bridge in + * src/codegen/zig.rs provides these `c_*` symbols, delegating to user code.) + */ +#include +#include + +#define ECHO_ITEMS 8 +#define ECHO_PAYLOAD 4 + +/* Lifecycle ---------------------------------------------------------------- */ +int c_init(void) { return 0; } +int c_shutdown(void) { return 0; } + +/* Data I/O ----------------------------------------------------------------- */ +int c_get_total_items(void) { return ECHO_ITEMS; } + +int c_load_item(int idx, unsigned char *buf, size_t *len) { + if (idx < 0 || idx >= ECHO_ITEMS) return 2; /* invalid_param */ + if (*len < ECHO_PAYLOAD) return 3; /* out_of_memory */ + for (size_t k = 0; k < ECHO_PAYLOAD; ++k) + buf[k] = (unsigned char)(idx + (int)k); + *len = ECHO_PAYLOAD; + return 0; +} + +int c_store_result(int idx, unsigned char *buf, size_t len) { + (void)idx; (void)buf; (void)len; + return 0; +} + +/* Processing --------------------------------------------------------------- */ +int c_process_item(unsigned char *in_buf, size_t in_len, + unsigned char *out_buf, size_t *out_len) { + memcpy(out_buf, in_buf, in_len); /* echo */ + *out_len = in_len; + return 0; +} + +int c_process_chunk(unsigned char *items_buf, int item_count, + int *item_offsets, int *item_sizes, + unsigned char *out_buf, size_t *out_len) { + (void)items_buf; (void)item_count; (void)item_offsets; (void)item_sizes; + (void)out_buf; + *out_len = 0; + return 0; +} + +/* Reduction ---------------------------------------------------------------- */ +int c_reduce(unsigned char *a_buf, size_t a_len, + unsigned char *b_buf, size_t b_len, + unsigned char *out_buf, size_t *out_len) { + memcpy(out_buf, a_buf, a_len); + memcpy(out_buf + a_len, b_buf, b_len); + *out_len = a_len + b_len; + return 0; +} + +/* Match predicate (for 'first' gather) ------------------------------------- */ +int c_is_match(unsigned char *buf, size_t len) { + (void)buf; (void)len; + return 0; +} + +/* Key hash (for keyed partition) ------------------------------------------- */ +unsigned int c_key_hash(unsigned char *buf, size_t len) { + (void)buf; (void)len; + return 0u; +} + +/* Checkpoint (optional — not implemented in the stub) ---------------------- */ +int c_checkpoint_save(unsigned char *buf, size_t len, const char *tag) { + (void)buf; (void)len; (void)tag; + return -1; +} + +int c_checkpoint_load(unsigned char *buf, size_t *len, const char *tag) { + (void)buf; (void)len; (void)tag; + return -1; +} diff --git a/examples/golden/generated/build.sh b/examples/golden/generated/build.sh new file mode 100755 index 0000000..3b4ffdb --- /dev/null +++ b/examples/golden/generated/build.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Auto-generated by Chapeliser — build script for workload: echo +# Regenerate with: chapeliser generate + +set -euo pipefail + +# Override these environment variables for your setup: +# ZIG — path to zig compiler (default: zig) +# CHPL — path to Chapel compiler (default: chpl) +# USER_LIB — path to user's compiled library (.a or .o) +# CHPL_FLAGS — extra Chapel compiler flags +ZIG=${ZIG:-zig} +CHPL=${CHPL:-chpl} +USER_LIB=${USER_LIB:-} +CHPL_FLAGS=${CHPL_FLAGS:-} + +echo "=== Chapeliser Build: echo ===" + +# Step 1: Compile the Zig FFI bridge to a C-ABI object file +echo "[1/3] Compiling Zig FFI bridge..." +$ZIG build-obj -O ReleaseFast zig/echo_ffi.zig -femit-bin=echo_ffi.o + +# Step 2: Link user's application code +echo "[2/3] Checking user code..." +if [ -z "$USER_LIB" ]; then + echo " WARNING: USER_LIB not set. You must provide your compiled application." + echo " For Rust: cargo build --release && export USER_LIB=target/release/libecho.a" + echo " For C: gcc -c -O2 your_code.c -o user_code.o && export USER_LIB=user_code.o" + echo " For Zig: zig build-obj -O ReleaseFast your_code.zig && export USER_LIB=your_code.o" + exit 1 +fi + +if [ ! -f "$USER_LIB" ]; then + echo " ERROR: USER_LIB=$USER_LIB not found" + exit 1 +fi +echo " Using: $USER_LIB" + +# Step 3: Compile the Chapel distributed wrapper +echo "[3/3] Compiling Chapel wrapper..." +$CHPL --fast -I include/ chapel/echo_distributed.chpl echo_ffi.o "$USER_LIB" $CHPL_FLAGS -o echo_distributed + +echo "=== Build complete: ./echo_distributed ===" +echo "Run with: ./echo_distributed -nl " +echo " --totalItems=N override item count" +echo " --maxItemBytes=N override max buffer size" +echo " --grainSize=N override chunk grain size" +echo " --maxRetries=N override retry limit" +echo " --enableCheckpoint enable checkpointing" diff --git a/examples/golden/generated/chapel/echo_distributed.chpl b/examples/golden/generated/chapel/echo_distributed.chpl new file mode 100644 index 0000000..c30bc74 --- /dev/null +++ b/examples/golden/generated/chapel/echo_distributed.chpl @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: MPL-2.0 +// Auto-generated by Chapeliser — do not edit manually. +// Workload: echo +// Partition: per-item, Gather: merge +// Serialization: raw, Max item bytes: 4096 +// Regenerate with: chapeliser generate + +module echo_Distributed { + + use CTypes; + use List; + use Time; + use IO; + + // ------------------------------------------------------------------ + // Runtime configuration (override with --name=value) + // ------------------------------------------------------------------ + + config const totalItems: int = 0; // 0 = auto-detect via c_get_total_items() + config const maxItemBytes: int = 4096; // max serialised size per item + config const grainSize: int = 1; // items per task (for chunk strategy) + config const maxRetries: int = 3; // per-item retry limit + config const enableCheckpoint: bool = false; // periodic checkpoint + config const checkpointIntervalSecs: int = 300; // seconds between checkpoints + + // ------------------------------------------------------------------ + // FFI declarations (implemented in Zig bridge / user code) + // ------------------------------------------------------------------ + + // Lifecycle: called once on locale 0 + extern proc c_init(): c_int; + extern proc c_shutdown(): c_int; + + // Data I/O: load/store serialised items on locale 0 + extern proc c_get_total_items(): c_int; + extern proc c_load_item(idx: c_int, buf: c_ptr(c_uchar), len: c_ptr(c_size_t)): c_int; + extern proc c_store_result(idx: c_int, buf: c_ptr(c_uchar), len: c_size_t): c_int; + + // Processing: called per item/chunk on any locale + extern proc c_process_item( + in_buf: c_ptr(c_uchar), in_len: c_size_t, + out_buf: c_ptr(c_uchar), out_len: c_ptr(c_size_t) + ): c_int; + + extern proc c_process_chunk( + items_buf: c_ptr(c_uchar), item_count: c_int, item_offsets: c_ptr(c_int), item_sizes: c_ptr(c_int), + out_buf: c_ptr(c_uchar), out_len: c_ptr(c_size_t) + ): c_int; + + // Reduction: combine two results into one + extern proc c_reduce( + a_buf: c_ptr(c_uchar), a_len: c_size_t, + b_buf: c_ptr(c_uchar), b_len: c_size_t, + out_buf: c_ptr(c_uchar), out_len: c_ptr(c_size_t) + ): c_int; + + // Match predicate: returns 1 if result satisfies search criterion + extern proc c_is_match(buf: c_ptr(c_uchar), len: c_size_t): c_int; + + // Key hash: returns hash of item's key for keyed distribution + extern proc c_key_hash(buf: c_ptr(c_uchar), len: c_size_t): c_uint; + + // Checkpoint: save/load progress (optional) + extern proc c_checkpoint_save( + buf: c_ptr(c_uchar), len: c_size_t, tag: c_ptrConst(c_char) + ): c_int; + extern proc c_checkpoint_load( + buf: c_ptr(c_uchar), len: c_ptr(c_size_t), tag: c_ptrConst(c_char) + ): c_int; + + // ------------------------------------------------------------------ + // Helper procedures + // ------------------------------------------------------------------ + + // Compute the item range assigned to a locale for even distribution. + proc localeRange(locId: int, nLocs: int, nItems: int): range { + const base = nItems / nLocs; + const rem = nItems % nLocs; + const lo = locId * base + min(locId, rem); + const hi = lo + base + (if locId < rem then 1 else 0); + return lo..#(hi - lo); + } + + // Process one item with retry. Returns 0 on success, last error code on failure. + proc processWithRetry(ref inBuf: [] uint(8), inLen: c_size_t, + ref outBuf: [] uint(8), ref outLen: c_size_t): c_int { + var lastRc: c_int = -1; + for attempt in 0..#maxRetries { + lastRc = c_process_item(c_ptrTo(inBuf[0]), inLen, c_ptrTo(outBuf[0]), c_ptrTo(outLen)); + if lastRc == 0 then return 0; + writeln(" WARN: item processing failed (attempt ", attempt + 1, "/", maxRetries, ", rc=", lastRc, ")"); + } + return lastRc; + } + + // ------------------------------------------------------------------ + // Main entry point + // ------------------------------------------------------------------ + + proc main() { + const t0 = timeSinceEpoch().totalSeconds(); + + writeln("Chapeliser [echo]: initialising on ", numLocales, " locale(s)"); + { + const rc = c_init(); + if rc != 0 { + writeln("FATAL: c_init() returned ", rc); + return; + } + } + + const nItems: int = if totalItems > 0 then totalItems else c_get_total_items(): int; + if nItems <= 0 { + writeln("Chapeliser: nothing to do (nItems=", nItems, ")"); + c_shutdown(); + return; + } + writeln("Chapeliser: distributing ", nItems, " items (partition=per-item, gather=merge)"); + + // ================================================================ + // Phase 1: Load items on locale 0 + // ================================================================ + + var itemData: [0..#nItems] [0..#maxItemBytes] uint(8); + var itemSizes: [0..#nItems] c_size_t; + + var loadFailed = false; + on Locales[0] { + for i in 0..#nItems { + var sz: c_size_t = maxItemBytes: c_size_t; + const rc = c_load_item(i: c_int, c_ptrTo(itemData[i][0]), c_ptrTo(sz)); + if rc != 0 { + writeln("FATAL: c_load_item(", i, ") returned ", rc); + loadFailed = true; + break; + } + itemSizes[i] = sz; + } + } + // 'return' is illegal inside an 'on' block; bail out after it. + if loadFailed { + c_shutdown(); + return; + } + writeln(" Loaded ", nItems, " items"); + + // ================================================================ + // Phase 2: Distribute & process (strategy: per-item) + // ================================================================ + + var resultData: [0..#nItems] [0..#maxItemBytes] uint(8); + var resultSizes: [0..#nItems] c_size_t; + var resultOk: [0..#nItems] bool; // track which items succeeded + + // Per-item: each locale gets nItems/numLocales items + coforall loc in Locales with (ref resultData, ref resultSizes, ref resultOk) do on loc { + const myRange = localeRange(loc.id, numLocales, nItems); + for i in myRange { + // Copy item to locale-local buffer, process, store result + var localIn: [0..#maxItemBytes] uint(8) = itemData[i]; + var localOut: [0..#maxItemBytes] uint(8); + var outLen: c_size_t = 0; + const rc = processWithRetry(localIn, itemSizes[i], localOut, outLen); + if rc == 0 { + resultData[i] = localOut; + resultSizes[i] = outLen; + resultOk[i] = true; + } else { + writeln(" ERROR: item ", i, " failed after ", maxRetries, " retries (rc=", rc, ")"); + resultOk[i] = false; + } + } + + // Checkpoint: save locale-local progress if enabled + if enableCheckpoint { + // Serialise locale's result indices into a compact buffer for checkpoint. + // The tag encodes the locale id so checkpoints don't collide. + var tag: string = "locale-" + loc.id:string; + var ckBuf: [0..#8] uint(8); // minimal: just store completion marker + ckBuf[0] = 1; // 1 = this locale finished + c_checkpoint_save(c_ptrTo(ckBuf[0]), 1: c_size_t, tag.c_str()); + } + } + + // ================================================================ + // Phase 3: Gather results (strategy: merge) + // ================================================================ + + // Merge: results already stored in resultData[i] by distribution phase. + // Count successes for reporting. + var nSuccess = + reduce resultOk:int; + writeln(" Gathered ", nSuccess, "/", nItems, " results (merge)"); + + // ================================================================ + // Phase 4: Store results via c_store_result on locale 0 + // ================================================================ + + on Locales[0] { + for i in 0..#nItems { + if !resultOk[i] then continue; + const rc = c_store_result(i: c_int, c_ptrTo(resultData[i][0]), resultSizes[i]); + if rc != 0 then writeln(" WARN: c_store_result(", i, ") returned ", rc); + } + } + + c_shutdown(); + const elapsed = timeSinceEpoch().totalSeconds() - t0; + writeln("Chapeliser [echo]: complete in ", elapsed:real(32), "s"); + } + +} diff --git a/examples/golden/generated/include/echo_chapeliser.h b/examples/golden/generated/include/echo_chapeliser.h new file mode 100644 index 0000000..02a5b5e --- /dev/null +++ b/examples/golden/generated/include/echo_chapeliser.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: MPL-2.0 */ +/* Auto-generated by Chapeliser — do not edit manually. */ +/* C header for workload: echo */ +/* Regenerate with: chapeliser generate */ + +#ifndef CHAPELISER_ECHO_H +#define CHAPELISER_ECHO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Chapeliser FFI contract for workload: echo + * + * Implement these functions with C linkage in your application. + * Chapeliser's generated Chapel + Zig code calls them at runtime. + * + * Return values: 0 = success, non-zero = error code. + * + * For Rust: #[no_mangle] pub extern "C" fn echo_init() -> i32 { ... } + * For Zig: export fn echo_init() callconv(.C) c_int { ... } + * For C: int echo_init(void) { ... } + */ + +/* ---- Lifecycle ---- */ + +/* Initialise the workload. Called once on locale 0 before processing. */ +int echo_init(void); + +/* Shut down the workload. Called once on locale 0 after all results stored. */ +int echo_shutdown(void); + +/* ---- Data I/O ---- */ + +/* Return the total number of input items. Called on locale 0. */ +int echo_get_total_items(void); + +/* Serialise input item `idx` into `buf`. Set `*len` to bytes written. */ +/* `buf` has capacity 4096 bytes (from manifest max-item-bytes). */ +int echo_load_item(int idx, uint8_t* buf, size_t* len); + +/* Receive a processed result. `buf` contains `len` serialised bytes. */ +int echo_store_result(int idx, const uint8_t* buf, size_t len); + +/* ---- Processing ---- */ + +/* Process a single serialised item. Read from in_buf/in_len, */ +/* write result to out_buf, set *out_len. Called on any locale. */ +int echo_process_item( + const uint8_t* in_buf, size_t in_len, + uint8_t* out_buf, size_t* out_len +); + +/* Process a chunk of items (for chunk partition strategy). */ +int echo_process_chunk( + const uint8_t* items_buf, int item_count, + const int* item_offsets, const int* item_sizes, + uint8_t* out_buf, size_t* out_len +); + +/* ---- Reduction (for reduce/tree-reduce gather) ---- */ + +/* Combine two results into one. */ +int echo_reduce( + const uint8_t* a_buf, size_t a_len, + const uint8_t* b_buf, size_t b_len, + uint8_t* out_buf, size_t* out_len +); + +/* ---- Match predicate (for 'first' gather) ---- */ + +/* Return 1 if result satisfies the search criterion, 0 otherwise. */ +int echo_is_match(const uint8_t* buf, size_t len); + +/* ---- Key hash (for keyed partition) ---- */ + +/* Return a hash of the item's key for distribution routing. */ +unsigned int echo_key_hash(const uint8_t* buf, size_t len); + +/* ---- Checkpoint (optional) ---- */ + +/* Save checkpoint data. Return -1 if not implemented. */ +int echo_checkpoint_save(const uint8_t* buf, size_t len, const char* tag); + +/* Load checkpoint data. Return -1 if not implemented. */ +int echo_checkpoint_load(uint8_t* buf, size_t* len, const char* tag); + +#ifdef __cplusplus +} +#endif + +#endif /* CHAPELISER_ECHO_H */ diff --git a/examples/golden/generated/zig/echo_ffi.zig b/examples/golden/generated/zig/echo_ffi.zig new file mode 100644 index 0000000..5e26871 --- /dev/null +++ b/examples/golden/generated/zig/echo_ffi.zig @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MPL-2.0 +// Auto-generated by Chapeliser — do not edit manually. +// Zig FFI bridge for workload: echo +// Regenerate with: chapeliser generate + +const std = @import("std"); + +// ---------------------------------------------------------------- +// User-provided symbols (implement these in your application) +// Link against your compiled code when building with Chapel. +// ---------------------------------------------------------------- + +// Lifecycle +extern fn echo_init() callconv(.C) c_int; +extern fn echo_shutdown() callconv(.C) c_int; + +// Data I/O +extern fn echo_get_total_items() callconv(.C) c_int; +extern fn echo_load_item(idx: c_int, buf: [*]u8, len: *usize) callconv(.C) c_int; +extern fn echo_store_result(idx: c_int, buf: [*]const u8, len: usize) callconv(.C) c_int; + +// Processing +extern fn echo_process_item(in_buf: [*]const u8, in_len: usize, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int; +extern fn echo_process_chunk(items_buf: [*]const u8, item_count: c_int, item_offsets: [*]const c_int, item_sizes: [*]const c_int, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int; + +// Reduction +extern fn echo_reduce(a_buf: [*]const u8, a_len: usize, b_buf: [*]const u8, b_len: usize, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int; + +// Match predicate +extern fn echo_is_match(buf: [*]const u8, len: usize) callconv(.C) c_int; + +// Key hash +extern fn echo_key_hash(buf: [*]const u8, len: usize) callconv(.C) c_uint; + +// Checkpoint (optional — return -1 if not implemented) +extern fn echo_checkpoint_save(buf: [*]const u8, len: usize, tag: [*:0]const u8) callconv(.C) c_int; +extern fn echo_checkpoint_load(buf: [*]u8, len: *usize, tag: [*:0]const u8) callconv(.C) c_int; + +// ---------------------------------------------------------------- +// Chapel-facing C-ABI exports +// These are called by the generated Chapel code via `extern proc`. +// ---------------------------------------------------------------- + +export fn c_init() callconv(.C) c_int { + return echo_init(); +} + +export fn c_shutdown() callconv(.C) c_int { + return echo_shutdown(); +} + +export fn c_get_total_items() callconv(.C) c_int { + return echo_get_total_items(); +} + +export fn c_load_item(idx: c_int, buf: [*]u8, len: *usize) callconv(.C) c_int { + return echo_load_item(idx, buf, len); +} + +export fn c_store_result(idx: c_int, buf: [*]const u8, len: usize) callconv(.C) c_int { + return echo_store_result(idx, buf, len); +} + +export fn c_process_item(in_buf: [*]const u8, in_len: usize, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int { + return echo_process_item(in_buf, in_len, out_buf, out_len); +} + +export fn c_process_chunk(items_buf: [*]const u8, item_count: c_int, item_offsets: [*]const c_int, item_sizes: [*]const c_int, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int { + return echo_process_chunk(items_buf, item_count, item_offsets, item_sizes, out_buf, out_len); +} + +export fn c_reduce(a_buf: [*]const u8, a_len: usize, b_buf: [*]const u8, b_len: usize, out_buf: [*]u8, out_len: *usize) callconv(.C) c_int { + return echo_reduce(a_buf, a_len, b_buf, b_len, out_buf, out_len); +} + +export fn c_is_match(buf: [*]const u8, len: usize) callconv(.C) c_int { + return echo_is_match(buf, len); +} + +export fn c_key_hash(buf: [*]const u8, len: usize) callconv(.C) c_uint { + return echo_key_hash(buf, len); +} + +export fn c_checkpoint_save(buf: [*]const u8, len: usize, tag: [*:0]const u8) callconv(.C) c_int { + return echo_checkpoint_save(buf, len, tag); +} + +export fn c_checkpoint_load(buf: [*]u8, len: *usize, tag: [*:0]const u8) callconv(.C) c_int { + return echo_checkpoint_load(buf, len, tag); +} diff --git a/src/codegen/chapel.rs b/src/codegen/chapel.rs index 097a7ea..8e2e5e1 100644 --- a/src/codegen/chapel.rs +++ b/src/codegen/chapel.rs @@ -442,6 +442,7 @@ fn write_load_phase(src: &mut String) -> Result<()> { writeln!(src, " var itemSizes: [0..#nItems] c_size_t;")?; writeln!(src)?; + writeln!(src, " var loadFailed = false;")?; writeln!(src, " on Locales[0] {{")?; writeln!(src, " for i in 0..#nItems {{")?; writeln!(src, " var sz: c_size_t = maxItemBytes: c_size_t;")?; @@ -454,12 +455,20 @@ fn write_load_phase(src: &mut String) -> Result<()> { src, " writeln(\"FATAL: c_load_item(\", i, \") returned \", rc);" )?; - writeln!(src, " c_shutdown();")?; - writeln!(src, " return;")?; + writeln!(src, " loadFailed = true;")?; + writeln!(src, " break;")?; writeln!(src, " }}")?; writeln!(src, " itemSizes[i] = sz;")?; writeln!(src, " }}")?; writeln!(src, " }}")?; + writeln!( + src, + " // 'return' is illegal inside an 'on' block; bail out after it." + )?; + writeln!(src, " if loadFailed {{")?; + writeln!(src, " c_shutdown();")?; + writeln!(src, " return;")?; + writeln!(src, " }}")?; writeln!(src, " writeln(\" Loaded \", nItems, \" items\");")?; writeln!(src)?; diff --git a/src/interface/abi/Foreign.idr b/src/interface/abi/Chapeliser/ABI/Foreign.idr similarity index 100% rename from src/interface/abi/Foreign.idr rename to src/interface/abi/Chapeliser/ABI/Foreign.idr diff --git a/src/interface/abi/Layout.idr b/src/interface/abi/Chapeliser/ABI/Layout.idr similarity index 100% rename from src/interface/abi/Layout.idr rename to src/interface/abi/Chapeliser/ABI/Layout.idr diff --git a/src/interface/abi/Types.idr b/src/interface/abi/Chapeliser/ABI/Types.idr similarity index 100% rename from src/interface/abi/Types.idr rename to src/interface/abi/Chapeliser/ABI/Types.idr diff --git a/src/interface/ffi/build.zig b/src/interface/ffi/build.zig index 89cd8d0..d1a8837 100644 --- a/src/interface/ffi/build.zig +++ b/src/interface/ffi/build.zig @@ -7,15 +7,17 @@ pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); - // Shared library (.so, .dylib, .dll) + // Shared library (.so, .dylib, .dll). + // NOTE: no explicit `.version` — a versioned shared library trips a null + // deref in Zig 0.14's InstallArtifact (major_only_filename); Chapel links + // the static archive anyway. const lib = b.addSharedLibrary(.{ .name = "chapeliser_ffi", .root_source_file = b.path("src/main.zig"), .target = target, .optimize = optimize, }); - - lib.version = .{ .major = 0, .minor = 1, .patch = 0 }; + lib.linkLibC(); // main.zig uses std.heap.c_allocator // Static library (.a) — Chapel links against this const lib_static = b.addStaticLibrary(.{ @@ -24,6 +26,7 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); + lib_static.linkLibC(); b.installArtifact(lib); b.installArtifact(lib_static); @@ -34,6 +37,7 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); + lib_tests.linkLibC(); const run_lib_tests = b.addRunArtifact(lib_tests); const test_step = b.step("test", "Run library tests"); @@ -45,7 +49,7 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); - + integration_tests.linkLibC(); integration_tests.linkLibrary(lib); const run_integration_tests = b.addRunArtifact(integration_tests);