Fix SIVACOR Part B generation for revisions

REPLICATION.md

@@ -411,6 +411,8 @@ List the files here, with a brief description of each.
 - MATLAB R2025b
 - Intel Compiler 3.14152 (note: there is no such thing, so please verify the version!)
 
+{{ sivacor-partb-computing-environment.md }}
+
 ## Replication steps
 
 
@@ -444,6 +446,8 @@ error: command distinct unknown
 - Code failed because of a typo in the name of the file "`superdata.dta`" (was: `superdta.dta`). Fixed. Ran again.
 - Code ran to completion. 
 
+{{ sivacor-partb-replication-steps.md }}
+
 > INSTRUCTIONS: ==>  Workflow stage: You are now going to *Writing Report*. Verify that both PartA and PartB have been completed.
 
 ## Findings
@@ -452,6 +456,7 @@ error: command distinct unknown
 
 > INSTRUCTIONS: Even when there is an external reproducibility report, summarize the findings here. 
 
+{{ sivacor-partb-findings.md }}
 
 ### Missing Requirements
 

automations/18_summarize_sivacor.sh

@@ -4,18 +4,38 @@
 
 set -euo pipefail
 
+PYTHON=${PYTHON:-}
+if [[ -z "$PYTHON" ]]
+then
+  if command -v python3 >/dev/null 2>&1
+  then
+    PYTHON=python3
+  elif command -v python >/dev/null 2>&1
+  then
+    PYTHON=python
+  else
+    echo "ERROR: Could not find python3 or python in PATH."
+    exit 1
+  fi
+elif ! command -v "$PYTHON" >/dev/null 2>&1
+then
+  echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found."
+  exit 1
+fi
+
 jsonld=""
 output_dir="generated"
-output="REPLICATION-PartB-SIVACOR.md"
+output=""
 report="REPLICATION-PartB.md"
+template="REPLICATION.md"
 dry_run=""
 replace_report=0
 
 usage() {
-  echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [--replace-report] [--dry-run]"
+  echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [-t template.md] [--replace-report] [--dry-run]"
   echo ""
   echo "If -j is omitted, the first */tro/tro-*.jsonld file is used."
-  echo "By default this writes REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md."
+  echo "By default this writes generated/REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md."
 }
 
 while [[ $# -gt 0 ]]
@@ -33,6 +53,10 @@ do
       report="$2"
       shift 2
       ;;
+    -t|--template)
+      template="$2"
+      shift 2
+      ;;
     -d|--output-dir)
       output_dir="$2"
       shift 2
@@ -68,20 +92,62 @@ then
   exit 1
 fi
 
-tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$report" --output "$output" $dry_run
+if [[ -z "$output" ]]
+then
+  output="$output_dir/REPLICATION-PartB-SIVACOR.md"
+fi
+
+if [[ ! -f "$template" ]]
+then
+  echo "ERROR: Template file '$template' not found."
+  exit 1
+fi
+
+PYTHON="$PYTHON" tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$template" --output "$output" $dry_run
 
 if [[ "$replace_report" -eq 1 ]]
 then
-  if [[ ! -f "$output" ]]
-  then
-    echo "ERROR: Generated SIVACOR Part B '$output' not found."
-    exit 1
-  fi
   if [[ "$dry_run" == "--dry-run" ]]
   then
-    echo "DRY RUN: Would copy '$output' to '$report'."
+    if [[ -f "$report" ]]
+    then
+      echo "DRY RUN: Would copy '$output' to '$report'."
+    elif [[ -f "REPLICATION.md" ]]
+    then
+      echo "DRY RUN: Would replace the Part B section of REPLICATION.md with '$output'."
+    else
+      echo "DRY RUN: Would copy '$output' to '$report'."
+    fi
   else
-    cp "$output" "$report"
-    echo "Copied '$output' to '$report'."
+    if [[ ! -f "$output" ]]
+    then
+      echo "ERROR: Generated SIVACOR Part B '$output' not found."
+      exit 1
+    fi
+    if [[ -f "$report" ]]
+    then
+      cp "$output" "$report"
+      echo "Copied '$output' to '$report'."
+    elif [[ -f "REPLICATION.md" ]]
+    then
+      splitline=$(grep -n "You are starting \*PartB\*." REPLICATION.md | cut -f1 -d: | head -1 || true)
+      if [[ -z "$splitline" ]]
+      then
+        splitline=$(grep -n "^## All data files provided$" REPLICATION.md | cut -f1 -d: | head -1 || true)
+      fi
+      if [[ -z "$splitline" ]]
+      then
+        echo "ERROR: Could not find the Part B split marker in REPLICATION.md."
+        exit 1
+      fi
+      tmp_report=$(mktemp)
+      head -n $(( splitline - 1 )) REPLICATION.md > "$tmp_report"
+      cat "$output" >> "$tmp_report"
+      mv "$tmp_report" REPLICATION.md
+      echo "Replaced the Part B section of REPLICATION.md with '$output'."
+    else
+      cp "$output" "$report"
+      echo "Copied '$output' to '$report'."
+    fi
   fi
 fi

automations/24_amend_report.sh

@@ -1,7 +1,26 @@
 #!/bin/bash
 #set -ev
 
-[[ "$SkipProcessing" == "yes" ]] && exit 0
+[[ "${SkipProcessing:-}" == "yes" ]] && exit 0
+
+PYTHON=${PYTHON:-}
+if [[ -z "$PYTHON" ]]
+then
+  if command -v python3 >/dev/null 2>&1
+  then
+    PYTHON=python3
+  elif command -v python >/dev/null 2>&1
+  then
+    PYTHON=python
+  else
+    echo "ERROR: Could not find python3 or python in PATH."
+    exit 1
+  fi
+elif ! command -v "$PYTHON" >/dev/null 2>&1
+then
+  echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found."
+  exit 1
+fi
 
 [[ -z $1 ]] && indir=generated || indir=$@
 
@@ -97,18 +116,29 @@ if [ ! -f "$indir/manifest.restricted.txt" ]; then
   echo "not present" > "$indir/manifest.restricted.txt"
 fi
 
+for sivacor_file in \
+  sivacor-partb-computing-environment.md \
+  sivacor-partb-replication-steps.md \
+  sivacor-partb-findings.md \
+  sivacor-partb-appendix.md
+do
+  if [ ! -f "$indir/$sivacor_file" ]; then
+    : > "$indir/$sivacor_file"
+  fi
+done
+
 
 
 # Now use the template to fill in the main part
 tmpmain=$(mktemp)
 tmpapp=$(mktemp)
 
-python3 tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain
+"$PYTHON" tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain
 
 # If the {{ large-file-report.md }} was not generated, remove the placeholder
 
-if [ ! -f "generated/large-file-report.md" ]; then
-  sed -i 's/{{ large-file-report.md }}/\n/' $tmpmain
+if [ ! -f "$indir/large-file-report.md" ]; then
+  "$PYTHON" -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text(encoding="utf-8").replace("{{ large-file-report.md }}", "\n"), encoding="utf-8")' "$tmpmain"
 fi
 
 # If there is a line with "Automatically Generated Appendices", we remove it and everything after it.
@@ -122,7 +152,7 @@ fi
 
 # Fill in the appendix
 
-python3 tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix
+"$PYTHON" tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix
 
 # DISABLED: Append the generated appendix to the base file
 echo "" >> $tmpapp

docs/96-91-pipeline-overview.md

@@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits
 
 **What it does**:
 - Finds the submitted TRO JSON-LD file
-- Generates `REPLICATION-PartB-SIVACOR.md`
-- Preserves the normal `REPLICATION-PartB.md` template structure
+- Generates `generated/REPLICATION-PartB-SIVACOR.md`
+- Extracts Part B from the single `REPLICATION.md` template during preprocessing
 - Inserts the SIVACOR execution environment into Computing Environment of the Replicator
 - Inserts what SIVACOR actually ran into Replication steps
 - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings
-- Adds the full SIVACOR arrangement comparison to the Appendix
-- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report`
+- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison
+- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report
 - Does not rerun author code
 - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification
 

docs/automations/index.md

@@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits
 
 **What it does**:
 - Finds the submitted TRO JSON-LD file
-- Generates `REPLICATION-PartB-SIVACOR.md`
-- Preserves the normal `REPLICATION-PartB.md` template structure
+- Generates `generated/REPLICATION-PartB-SIVACOR.md`
+- Extracts Part B from the single `REPLICATION.md` template during preprocessing
 - Inserts the SIVACOR execution environment into Computing Environment of the Replicator
 - Inserts what SIVACOR actually ran into Replication steps
 - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings
-- Adds the full SIVACOR arrangement comparison to the Appendix
-- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report`
+- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison
+- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report
 - Does not rerun author code
 - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification
 

docs/tools/index.md

@@ -53,7 +53,7 @@ Python script for extracting computing environment, timing information, and SIVA
 **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/get_sivacor_info.py) | [Help](repository/96-90-get_sivacor_info.md)
 
 ### generate_sivacor_partb.sh
-Bash wrapper for generating `REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file preserves the normal Part B template structure and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings.
+Bash wrapper for generating `generated/REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file extracts Part B from the single `REPLICATION.md` template and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. It also writes `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template.
 
 **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/generate_sivacor_partb.sh) | [Help](repository/96-90-get_sivacor_info.md)
 

docs/tools/repository/96-90-get_sivacor_info.md

@@ -16,21 +16,21 @@ Parses SIVACOR JSONLD (TRO - Transparent Research Object) files to extract compu
 
 ```bash
 # Output to stdout
-python3.12 tools/get_sivacor_info.py <jsonld_file> <keyword>
-python3.12 tools/get_sivacor_info.py --jsonld <file> --key <keyword>
-python3.12 tools/get_sivacor_info.py --jobid <job_id> --key <keyword>
+python3 tools/get_sivacor_info.py <jsonld_file> <keyword>
+python3 tools/get_sivacor_info.py --jsonld <file> --key <keyword>
+python3 tools/get_sivacor_info.py --jobid <job_id> --key <keyword>
 
 # Update report file
-python3.12 tools/get_sivacor_info.py --jobid <job_id> --key <keyword> --report <report_file>
+python3 tools/get_sivacor_info.py --jobid <job_id> --key <keyword> --report <report_file>
 
 # Dry-run (preview without updating)
-python3.12 tools/get_sivacor_info.py --jobid <job_id> --key <keyword> --report <report_file> --dry-run
+python3 tools/get_sivacor_info.py --jobid <job_id> --key <keyword> --report <report_file> --dry-run
 
 # Generate SIVACOR Part B insert snippets from the TRO
-python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/partb-SIVACOR-computing-environment.md
-python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/partb-SIVACOR-replication-steps.md
-python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/partb-SIVACOR-findings.md
-python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/partb-SIVACOR-appendix.md
+python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/sivacor-partb-computing-environment.md
+python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/sivacor-partb-replication-steps.md
+python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/sivacor-partb-findings.md
+python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/sivacor-partb-appendix.md
 
 # Generate a template-consistent SIVACOR Part B file
 tools/generate_sivacor_partb.sh --dry-run
@@ -114,46 +114,46 @@ Generates the full SIVACOR arrangement comparison for the Appendix:
 - Lists other generated output, data/intermediate, log, R environment, and uncategorized paths
 - Lists removed and modified paths, if any
 
-`tools/generate_sivacor_partb.sh` combines these snippets with the existing `REPLICATION-PartB.md` template and writes `REPLICATION-PartB-SIVACOR.md`. `automations/18_summarize_sivacor.sh --replace-report` then copies that generated file over `REPLICATION-PartB.md`.
+`tools/generate_sivacor_partb.sh` extracts Part B from the single `REPLICATION.md` template, fills its SIVACOR placeholders, and writes `generated/REPLICATION-PartB-SIVACOR.md`. For non-SIVACOR reports, normal preprocessing replaces those placeholders with empty content. The SIVACOR appendix snippet is written to `generated/sivacor-partb-appendix.md` and included when the normal appendix template is regenerated. `automations/18_summarize_sivacor.sh --replace-report` then copies the generated Part B file over `REPLICATION-PartB.md`, or replaces the Part B section inside `REPLICATION.md` when a revision report is not split.
 
 ### Examples
 
 ```bash
 # Extract computing info and print to stdout
-python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing
+python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing
 
 # Extract timing info and print to stdout
-python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time
+python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time
 
 # Generate a template-consistent SIVACOR Part B file
 tools/generate_sivacor_partb.sh
 
 # Preview what would be added to report (dry-run)
-python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run
+python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run
 
 # Preview SIVACOR Part B generation
 ./automations/18_summarize_sivacor.sh --dry-run
 
 # Add computing info to report
-python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md
+python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md
 
 # Add timing info to report
-python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md
+python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md
 
-# Generate REPLICATION-PartB-SIVACOR.md, then copy it over REPLICATION-PartB.md
+# Generate generated/REPLICATION-PartB-SIVACOR.md, then apply it to the current report
 ./automations/18_summarize_sivacor.sh --replace-report
 
 # Using positional arguments
 cd 246302
-python3.12 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing
+python3 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing
 
 # Using file path directly
-python3.12 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time
+python3 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time
 ```
 
 ## SIVACOR Workflow Note
 
-For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `REPLICATION-PartB-SIVACOR.md`, then copy it over `REPLICATION-PartB.md` when ready. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," a concise SIVACOR-generated file summary into "Findings," and the full arrangement comparison into the Appendix. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification.
+For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `generated/REPLICATION-PartB-SIVACOR.md`, then apply it with `--replace-report` when ready. In split-report cases, this updates `REPLICATION-PartB.md`; in single-file revision reports, it replaces the Part B section inside `REPLICATION.md`. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," and a concise SIVACOR-generated file summary into "Findings." The full arrangement comparison is written to `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification.
 
 ## Requirements
 

template/REPLICATION_appendix.md

@@ -61,6 +61,8 @@ Some references for good commenting practices:
 
 {{ manifest-comparison.md }}
 
+{{ sivacor-partb-appendix.md }}
+
 ## Appendix: Not-for-publication data provided (if any)
 
 ```