diff --git a/REPLICATION.md b/REPLICATION.md index 4117a50..ddcaf7c 100644 --- a/REPLICATION.md +++ b/REPLICATION.md @@ -411,6 +411,8 @@ List the files here, with a brief description of each. - MATLAB R2025b - Intel Compiler 3.14152 (note: there is no such thing, so please verify the version!) +{{ sivacor-partb-computing-environment.md }} + ## Replication steps @@ -444,6 +446,8 @@ error: command distinct unknown - Code failed because of a typo in the name of the file "`superdata.dta`" (was: `superdta.dta`). Fixed. Ran again. - Code ran to completion. +{{ sivacor-partb-replication-steps.md }} + > INSTRUCTIONS: ==> Workflow stage: You are now going to *Writing Report*. Verify that both PartA and PartB have been completed. ## Findings @@ -452,6 +456,7 @@ error: command distinct unknown > INSTRUCTIONS: Even when there is an external reproducibility report, summarize the findings here. +{{ sivacor-partb-findings.md }} ### Missing Requirements diff --git a/automations/18_summarize_sivacor.sh b/automations/18_summarize_sivacor.sh index 5f204fd..d7cffd7 100755 --- a/automations/18_summarize_sivacor.sh +++ b/automations/18_summarize_sivacor.sh @@ -4,18 +4,38 @@ set -euo pipefail +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi + jsonld="" output_dir="generated" -output="REPLICATION-PartB-SIVACOR.md" +output="" report="REPLICATION-PartB.md" +template="REPLICATION.md" dry_run="" replace_report=0 usage() { - echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [--replace-report] [--dry-run]" + echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [-t template.md] [--replace-report] [--dry-run]" echo "" echo "If -j is omitted, the first */tro/tro-*.jsonld file is used." - echo "By default this writes REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md." + echo "By default this writes generated/REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md." } while [[ $# -gt 0 ]] @@ -33,6 +53,10 @@ do report="$2" shift 2 ;; + -t|--template) + template="$2" + shift 2 + ;; -d|--output-dir) output_dir="$2" shift 2 @@ -68,20 +92,62 @@ then exit 1 fi -tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$report" --output "$output" $dry_run +if [[ -z "$output" ]] +then + output="$output_dir/REPLICATION-PartB-SIVACOR.md" +fi + +if [[ ! -f "$template" ]] +then + echo "ERROR: Template file '$template' not found." + exit 1 +fi + +PYTHON="$PYTHON" tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$template" --output "$output" $dry_run if [[ "$replace_report" -eq 1 ]] then - if [[ ! -f "$output" ]] - then - echo "ERROR: Generated SIVACOR Part B '$output' not found." - exit 1 - fi if [[ "$dry_run" == "--dry-run" ]] then - echo "DRY RUN: Would copy '$output' to '$report'." + if [[ -f "$report" ]] + then + echo "DRY RUN: Would copy '$output' to '$report'." + elif [[ -f "REPLICATION.md" ]] + then + echo "DRY RUN: Would replace the Part B section of REPLICATION.md with '$output'." + else + echo "DRY RUN: Would copy '$output' to '$report'." + fi else - cp "$output" "$report" - echo "Copied '$output' to '$report'." + if [[ ! -f "$output" ]] + then + echo "ERROR: Generated SIVACOR Part B '$output' not found." + exit 1 + fi + if [[ -f "$report" ]] + then + cp "$output" "$report" + echo "Copied '$output' to '$report'." + elif [[ -f "REPLICATION.md" ]] + then + splitline=$(grep -n "You are starting \*PartB\*." REPLICATION.md | cut -f1 -d: | head -1 || true) + if [[ -z "$splitline" ]] + then + splitline=$(grep -n "^## All data files provided$" REPLICATION.md | cut -f1 -d: | head -1 || true) + fi + if [[ -z "$splitline" ]] + then + echo "ERROR: Could not find the Part B split marker in REPLICATION.md." + exit 1 + fi + tmp_report=$(mktemp) + head -n $(( splitline - 1 )) REPLICATION.md > "$tmp_report" + cat "$output" >> "$tmp_report" + mv "$tmp_report" REPLICATION.md + echo "Replaced the Part B section of REPLICATION.md with '$output'." + else + cp "$output" "$report" + echo "Copied '$output' to '$report'." + fi fi fi diff --git a/automations/24_amend_report.sh b/automations/24_amend_report.sh index 4417191..0641cc3 100755 --- a/automations/24_amend_report.sh +++ b/automations/24_amend_report.sh @@ -1,7 +1,26 @@ #!/bin/bash #set -ev -[[ "$SkipProcessing" == "yes" ]] && exit 0 +[[ "${SkipProcessing:-}" == "yes" ]] && exit 0 + +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi [[ -z $1 ]] && indir=generated || indir=$@ @@ -97,18 +116,29 @@ if [ ! -f "$indir/manifest.restricted.txt" ]; then echo "not present" > "$indir/manifest.restricted.txt" fi +for sivacor_file in \ + sivacor-partb-computing-environment.md \ + sivacor-partb-replication-steps.md \ + sivacor-partb-findings.md \ + sivacor-partb-appendix.md +do + if [ ! -f "$indir/$sivacor_file" ]; then + : > "$indir/$sivacor_file" + fi +done + # Now use the template to fill in the main part tmpmain=$(mktemp) tmpapp=$(mktemp) -python3 tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain +"$PYTHON" tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain # If the {{ large-file-report.md }} was not generated, remove the placeholder -if [ ! -f "generated/large-file-report.md" ]; then - sed -i 's/{{ large-file-report.md }}/\n/' $tmpmain +if [ ! -f "$indir/large-file-report.md" ]; then + "$PYTHON" -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text(encoding="utf-8").replace("{{ large-file-report.md }}", "\n"), encoding="utf-8")' "$tmpmain" fi # If there is a line with "Automatically Generated Appendices", we remove it and everything after it. @@ -122,7 +152,7 @@ fi # Fill in the appendix -python3 tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix +"$PYTHON" tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix # DISABLED: Append the generated appendix to the base file echo "" >> $tmpapp diff --git a/docs/96-91-pipeline-overview.md b/docs/96-91-pipeline-overview.md index cf343db..0a939db 100644 --- a/docs/96-91-pipeline-overview.md +++ b/docs/96-91-pipeline-overview.md @@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file -- Generates `REPLICATION-PartB-SIVACOR.md` -- Preserves the normal `REPLICATION-PartB.md` template structure +- Generates `generated/REPLICATION-PartB-SIVACOR.md` +- Extracts Part B from the single `REPLICATION.md` template during preprocessing - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings -- Adds the full SIVACOR arrangement comparison to the Appendix -- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report` +- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison +- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report - Does not rerun author code - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification diff --git a/docs/automations/index.md b/docs/automations/index.md index cf343db..0a939db 100644 --- a/docs/automations/index.md +++ b/docs/automations/index.md @@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file -- Generates `REPLICATION-PartB-SIVACOR.md` -- Preserves the normal `REPLICATION-PartB.md` template structure +- Generates `generated/REPLICATION-PartB-SIVACOR.md` +- Extracts Part B from the single `REPLICATION.md` template during preprocessing - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings -- Adds the full SIVACOR arrangement comparison to the Appendix -- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report` +- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison +- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report - Does not rerun author code - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification diff --git a/docs/tools/index.md b/docs/tools/index.md index 1722914..6cb5d72 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -53,7 +53,7 @@ Python script for extracting computing environment, timing information, and SIVA **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/get_sivacor_info.py) | [Help](repository/96-90-get_sivacor_info.md) ### generate_sivacor_partb.sh -Bash wrapper for generating `REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file preserves the normal Part B template structure and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. +Bash wrapper for generating `generated/REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file extracts Part B from the single `REPLICATION.md` template and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. It also writes `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/generate_sivacor_partb.sh) | [Help](repository/96-90-get_sivacor_info.md) diff --git a/docs/tools/repository/96-90-get_sivacor_info.md b/docs/tools/repository/96-90-get_sivacor_info.md index dc89d93..2d3d1fc 100644 --- a/docs/tools/repository/96-90-get_sivacor_info.md +++ b/docs/tools/repository/96-90-get_sivacor_info.md @@ -16,21 +16,21 @@ Parses SIVACOR JSONLD (TRO - Transparent Research Object) files to extract compu ```bash # Output to stdout -python3.12 tools/get_sivacor_info.py -python3.12 tools/get_sivacor_info.py --jsonld --key -python3.12 tools/get_sivacor_info.py --jobid --key +python3 tools/get_sivacor_info.py +python3 tools/get_sivacor_info.py --jsonld --key +python3 tools/get_sivacor_info.py --jobid --key # Update report file -python3.12 tools/get_sivacor_info.py --jobid --key --report +python3 tools/get_sivacor_info.py --jobid --key --report # Dry-run (preview without updating) -python3.12 tools/get_sivacor_info.py --jobid --key --report --dry-run +python3 tools/get_sivacor_info.py --jobid --key --report --dry-run # Generate SIVACOR Part B insert snippets from the TRO -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/partb-SIVACOR-computing-environment.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/partb-SIVACOR-replication-steps.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/partb-SIVACOR-findings.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/partb-SIVACOR-appendix.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/sivacor-partb-computing-environment.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/sivacor-partb-replication-steps.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/sivacor-partb-findings.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/sivacor-partb-appendix.md # Generate a template-consistent SIVACOR Part B file tools/generate_sivacor_partb.sh --dry-run @@ -114,46 +114,46 @@ Generates the full SIVACOR arrangement comparison for the Appendix: - Lists other generated output, data/intermediate, log, R environment, and uncategorized paths - Lists removed and modified paths, if any -`tools/generate_sivacor_partb.sh` combines these snippets with the existing `REPLICATION-PartB.md` template and writes `REPLICATION-PartB-SIVACOR.md`. `automations/18_summarize_sivacor.sh --replace-report` then copies that generated file over `REPLICATION-PartB.md`. +`tools/generate_sivacor_partb.sh` extracts Part B from the single `REPLICATION.md` template, fills its SIVACOR placeholders, and writes `generated/REPLICATION-PartB-SIVACOR.md`. For non-SIVACOR reports, normal preprocessing replaces those placeholders with empty content. The SIVACOR appendix snippet is written to `generated/sivacor-partb-appendix.md` and included when the normal appendix template is regenerated. `automations/18_summarize_sivacor.sh --replace-report` then copies the generated Part B file over `REPLICATION-PartB.md`, or replaces the Part B section inside `REPLICATION.md` when a revision report is not split. ### Examples ```bash # Extract computing info and print to stdout -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing # Extract timing info and print to stdout -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time # Generate a template-consistent SIVACOR Part B file tools/generate_sivacor_partb.sh # Preview what would be added to report (dry-run) -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run # Preview SIVACOR Part B generation ./automations/18_summarize_sivacor.sh --dry-run # Add computing info to report -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md # Add timing info to report -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md -# Generate REPLICATION-PartB-SIVACOR.md, then copy it over REPLICATION-PartB.md +# Generate generated/REPLICATION-PartB-SIVACOR.md, then apply it to the current report ./automations/18_summarize_sivacor.sh --replace-report # Using positional arguments cd 246302 -python3.12 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing +python3 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing # Using file path directly -python3.12 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time +python3 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time ``` ## SIVACOR Workflow Note -For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `REPLICATION-PartB-SIVACOR.md`, then copy it over `REPLICATION-PartB.md` when ready. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," a concise SIVACOR-generated file summary into "Findings," and the full arrangement comparison into the Appendix. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification. +For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `generated/REPLICATION-PartB-SIVACOR.md`, then apply it with `--replace-report` when ready. In split-report cases, this updates `REPLICATION-PartB.md`; in single-file revision reports, it replaces the Part B section inside `REPLICATION.md`. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," and a concise SIVACOR-generated file summary into "Findings." The full arrangement comparison is written to `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification. ## Requirements diff --git a/template/REPLICATION_appendix.md b/template/REPLICATION_appendix.md index f684c6d..d1f9c77 100644 --- a/template/REPLICATION_appendix.md +++ b/template/REPLICATION_appendix.md @@ -61,6 +61,8 @@ Some references for good commenting practices: {{ manifest-comparison.md }} +{{ sivacor-partb-appendix.md }} + ## Appendix: Not-for-publication data provided (if any) ``` diff --git a/tools/generate_sivacor_partb.sh b/tools/generate_sivacor_partb.sh index 4dd81d8..6b69da2 100755 --- a/tools/generate_sivacor_partb.sh +++ b/tools/generate_sivacor_partb.sh @@ -3,27 +3,44 @@ set -euo pipefail +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi + jsonld="" output_dir="generated" -template="REPLICATION-PartB.md" -output="REPLICATION-PartB-SIVACOR.md" +template="REPLICATION.md" +output="" steps_output="" findings_output="" environment_output="" appendix_output="" -report_appendix="" usage() { echo "Usage: $0 [-j tro.jsonld] [-t template.md] [-o output.md] [-d output_dir] [--dry-run]" echo "" echo "If -j is omitted, the first */tro/tro-*.jsonld file is used." echo "Outputs:" - echo " generated/partb-SIVACOR-computing-environment.md" - echo " generated/partb-SIVACOR-replication-steps.md" - echo " generated/partb-SIVACOR-findings.md" - echo " generated/partb-SIVACOR-appendix.md" - echo " generated/REPLICATION_appendix.md" - echo " REPLICATION-PartB-SIVACOR.md" + echo " generated/sivacor-partb-computing-environment.md" + echo " generated/sivacor-partb-replication-steps.md" + echo " generated/sivacor-partb-findings.md" + echo " generated/sivacor-partb-appendix.md" + echo " generated/REPLICATION-PartB-SIVACOR.md" } dry_run="" @@ -74,39 +91,58 @@ then exit 1 fi +if [[ -z "$output" ]] +then + output="$output_dir/REPLICATION-PartB-SIVACOR.md" +fi + if [[ ! -f "$template" ]] then echo "ERROR: Template file '$template' not found." exit 1 fi -steps_output="$output_dir/partb-SIVACOR-replication-steps.md" -findings_output="$output_dir/partb-SIVACOR-findings.md" -environment_output="$output_dir/partb-SIVACOR-computing-environment.md" -appendix_output="$output_dir/partb-SIVACOR-appendix.md" -report_appendix="$output_dir/REPLICATION_appendix.md" - +steps_output="$output_dir/sivacor-partb-replication-steps.md" +findings_output="$output_dir/sivacor-partb-findings.md" +environment_output="$output_dir/sivacor-partb-computing-environment.md" +appendix_output="$output_dir/sivacor-partb-appendix.md" if [[ "$dry_run" != "--dry-run" ]] then mkdir -p "$output_dir" fi -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-computing-environment --output "$environment_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-replication-steps --output "$steps_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-findings --output "$findings_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-appendix --output "$appendix_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-computing-environment --output "$environment_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-replication-steps --output "$steps_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-findings --output "$findings_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-appendix --output "$appendix_output" $dry_run + +template_input="$template" +tmp_template="" +if grep -Fq "You are starting *PartB*." "$template" +then + splitline=$(grep -Fn "You are starting *PartB*." "$template" | cut -f1 -d: | head -1) + tmp_template=$(mktemp) + tail -n +"$splitline" "$template" > "$tmp_template" + template_input="$tmp_template" +fi if [[ "$dry_run" == "--dry-run" ]] then if [[ -f "$environment_output" && -f "$steps_output" && -f "$findings_output" && -f "$appendix_output" ]] then - python3 tools/insert_sivacor_partb_sections.py --template "$template" --computing-environment "$environment_output" --replication-steps "$steps_output" --findings "$findings_output" --output "$output" --dry-run - python3 tools/update_sivacor_appendix.py --appendix "$report_appendix" --sivacor-appendix "$appendix_output" --dry-run + preview_output=$(mktemp) + "$PYTHON" tools/replace_placeholders.py --infile "$template_input" --indir "$output_dir" --outfile "$preview_output" + cat "$preview_output" + rm "$preview_output" else echo "DRY RUN: Would combine generated snippets with '$template' into '$output'." - echo "DRY RUN: Would add '$appendix_output' to '$report_appendix'." + echo "DRY RUN: Would write the SIVACOR appendix snippet to '$appendix_output'." fi else - python3 tools/insert_sivacor_partb_sections.py --template "$template" --computing-environment "$environment_output" --replication-steps "$steps_output" --findings "$findings_output" --output "$output" - python3 tools/update_sivacor_appendix.py --appendix "$report_appendix" --sivacor-appendix "$appendix_output" + "$PYTHON" tools/replace_placeholders.py --infile "$template_input" --indir "$output_dir" --outfile "$output" +fi + +if [[ -n "$tmp_template" ]] +then + rm "$tmp_template" fi diff --git a/tools/get_sivacor_info.py b/tools/get_sivacor_info.py index 6fc7fa8..ff3c26e 100755 --- a/tools/get_sivacor_info.py +++ b/tools/get_sivacor_info.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.12 +#!/usr/bin/env python3 """ Parse SIVACOR JSONLD files for information. @@ -1171,7 +1171,7 @@ def main(): %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key computing %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key time %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb - %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb-sivacor --output REPLICATION-PartB-SIVACOR.md + %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb-sivacor --output generated/REPLICATION-PartB-SIVACOR.md %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md --dry-run """ diff --git a/tools/insert_sivacor_partb_sections.py b/tools/insert_sivacor_partb_sections.py deleted file mode 100644 index ff9176a..0000000 --- a/tools/insert_sivacor_partb_sections.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -"""Insert SIVACOR sections into an existing Part B Markdown template.""" - -import argparse -import re -import sys - - -def read_file(path): - with open(path, "r", encoding="utf-8") as handle: - return handle.read() - - -def write_file(path, content): - with open(path, "w", encoding="utf-8") as handle: - handle.write(content.rstrip()) - handle.write("\n") - - -def insert_after_replication_steps(content, insert_text): - marker = "## Replication steps" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Replication steps' section") - - next_heading = content.find("\n## ", pos + len(marker)) - section_end = next_heading if next_heading != -1 else len(content) - replacement = marker + "\n\n" + insert_text.strip() + "\n\n" - return content[:pos] + replacement + content[section_end:].lstrip() - - -def replace_computing_environment(content, insert_text): - marker = "## Computing Environment of the Replicator" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Computing Environment of the Replicator' section") - - next_heading = content.find("\n## ", pos + len(marker)) - section_end = next_heading if next_heading != -1 else len(content) - replacement = marker + "\n\n" + insert_text.strip() + "\n\n" - return content[:pos] + replacement + content[section_end:].lstrip() - - -def insert_after_findings_intro(content, insert_text): - marker = "## Findings" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Findings' section") - - first_subheading = content.find("\n### ", pos + len(marker)) - if first_subheading == -1: - raise ValueError("Could not find first Findings subsection") - - return content[:first_subheading].rstrip() + "\n\n" + insert_text + "\n\n" + content[first_subheading:].lstrip() - - -def remove_existing_sivacor_sections(content): - content = re.sub( - r"\nThe AEA reviewer did not rerun the author code\. The SIVACOR TRO records the following execution environment:\n\n.*?(?=\n## Replication steps)", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n\*\*SIVACOR automated execution summary\*\*.*?(?=\n- \[ \] The reproducibility check|\n> INSTRUCTIONS: provide details|\n## Findings)", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n### SIVACOR-generated files\n\n.*?(?=\n### |\n## )", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n## Appendix\n\n### SIVACOR arrangement comparison\n\n.*\Z", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n## Appendix: SIVACOR arrangement comparison\n\n.*\Z", - "\n", - content, - flags=re.DOTALL, - ) - return content - - -def main(): - parser = argparse.ArgumentParser(description="Insert generated SIVACOR Part B sections into a Markdown template.") - parser.add_argument("--template", required=True, help="Existing Part B Markdown template") - parser.add_argument("--computing-environment", required=True, help="Generated SIVACOR computing environment Markdown") - parser.add_argument("--replication-steps", required=True, help="Generated SIVACOR replication steps Markdown") - parser.add_argument("--findings", required=True, help="Generated SIVACOR findings Markdown") - parser.add_argument("--output", required=True, help="Output Markdown file") - parser.add_argument("--dry-run", action="store_true", help="Print output instead of writing") - args = parser.parse_args() - - content = remove_existing_sivacor_sections(read_file(args.template)) - content = replace_computing_environment(content, read_file(args.computing_environment).strip()) - content = insert_after_replication_steps(content, read_file(args.replication_steps).strip()) - content = insert_after_findings_intro(content, read_file(args.findings).strip()) - - if args.dry_run: - print(content) - else: - write_file(args.output, content) - print(f"Successfully wrote {args.output}.", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/tools/replace_placeholders.py b/tools/replace_placeholders.py index 12e3750..35c65ac 100755 --- a/tools/replace_placeholders.py +++ b/tools/replace_placeholders.py @@ -2,7 +2,11 @@ import os import shutil import argparse -import chardet + +try: + import chardet +except ImportError: + chardet = None TEMPLATE='REPLICATION.md' @@ -34,9 +38,13 @@ def replace_content(template,replacement,tag): # Detect encoding automatically with open(filepath, 'rb') as f: rawdata = f.read() - detected = chardet.detect(rawdata) - encoding = detected['encoding'] or 'utf-8' # Default to utf-8 if detection fails - confidence = detected['confidence'] + if chardet: + detected = chardet.detect(rawdata) + encoding = detected['encoding'] or 'utf-8' # Default to utf-8 if detection fails + confidence = detected['confidence'] + else: + encoding = 'utf-8' + confidence = 1.0 # Warn if encoding is not UTF-8 or confidence is low if encoding.lower() not in ['utf-8', 'utf-8-sig', 'ascii']: diff --git a/tools/update_sivacor_appendix.py b/tools/update_sivacor_appendix.py deleted file mode 100644 index 1af770b..0000000 --- a/tools/update_sivacor_appendix.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -"""Insert SIVACOR appendix material into generated/REPLICATION_appendix.md.""" - -import argparse -import os -import re -import sys - - -SECTION_PATTERN = re.compile( - r"\n## Appendix: SIVACOR arrangement comparison\n\n.*?(?=\n## Appendix: |\Z)", - flags=re.DOTALL, -) - - -def read_file(path): - with open(path, "r", encoding="utf-8") as handle: - return handle.read() - - -def write_file(path, content): - with open(path, "w", encoding="utf-8") as handle: - handle.write(content) - - -def normalize_heading(content): - return re.sub(r"\A### SIVACOR arrangement comparison", "## Appendix: SIVACOR arrangement comparison", content.strip()) - - -def update_appendix(existing_content, sivacor_content): - section = normalize_heading(sivacor_content) - if existing_content.strip(): - base = SECTION_PATTERN.sub("", existing_content).rstrip() - return base + "\n\n" + section + "\n" - return "# Automatically Generated Appendices\n\n> No action is required, unless indicated in the main body of the report.\n\n" + section + "\n" - - -def main(): - parser = argparse.ArgumentParser(description="Add SIVACOR arrangement comparison to generated appendix.") - parser.add_argument("--appendix", default="generated/REPLICATION_appendix.md", help="Generated appendix file to update") - parser.add_argument("--sivacor-appendix", required=True, help="Generated SIVACOR appendix snippet") - parser.add_argument("--dry-run", action="store_true", help="Print output instead of writing") - args = parser.parse_args() - - sivacor_content = read_file(args.sivacor_appendix) - existing_content = read_file(args.appendix) if os.path.exists(args.appendix) else "" - updated = update_appendix(existing_content, sivacor_content) - - if args.dry_run: - print(updated) - else: - os.makedirs(os.path.dirname(args.appendix) or ".", exist_ok=True) - write_file(args.appendix, updated) - print(f"Successfully updated {args.appendix}.", file=sys.stderr) - - -if __name__ == "__main__": - main()