From 1cb248f114a2bd388d4473a67f54a22d4def1518 Mon Sep 17 00:00:00 2001 From: dmcguir5 Date: Wed, 17 Jun 2026 19:12:01 -0400 Subject: [PATCH 1/3] Fix SIVACOR Part B generation for revisions --- automations/18_summarize_sivacor.sh | 111 +++++++- automations/24_amend_report.sh | 35 ++- docs/96-91-pipeline-overview.md | 8 +- docs/automations/index.md | 8 +- docs/tools/index.md | 2 +- .../repository/96-90-get_sivacor_info.md | 14 +- template/REPLICATION-PartB-sivacor.md | 257 ++++++++++++++++++ template/REPLICATION_appendix.md | 2 + tools/create_sivacor_partb_template.py | 72 +++++ tools/generate_sivacor_partb.sh | 69 +++-- tools/get_sivacor_info.py | 2 +- tools/insert_sivacor_partb_sections.py | 114 -------- tools/replace_placeholders.py | 16 +- tools/update_sivacor_appendix.py | 58 ---- 14 files changed, 538 insertions(+), 230 deletions(-) create mode 100644 template/REPLICATION-PartB-sivacor.md create mode 100644 tools/create_sivacor_partb_template.py delete mode 100644 tools/insert_sivacor_partb_sections.py delete mode 100644 tools/update_sivacor_appendix.py diff --git a/automations/18_summarize_sivacor.sh b/automations/18_summarize_sivacor.sh index 5f204fd..8c6f5dd 100755 --- a/automations/18_summarize_sivacor.sh +++ b/automations/18_summarize_sivacor.sh @@ -4,18 +4,38 @@ set -euo pipefail +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi + jsonld="" output_dir="generated" -output="REPLICATION-PartB-SIVACOR.md" +output="" report="REPLICATION-PartB.md" +template="template/REPLICATION-PartB-sivacor.md" dry_run="" replace_report=0 usage() { - echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [--replace-report] [--dry-run]" + echo "Usage: $0 [-j tro.jsonld] [-d output_dir] [-o output.md] [-r report.md] [-t template.md] [--replace-report] [--dry-run]" echo "" echo "If -j is omitted, the first */tro/tro-*.jsonld file is used." - echo "By default this writes REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md." + echo "By default this writes generated/REPLICATION-PartB-SIVACOR.md and does not modify REPLICATION-PartB.md." } while [[ $# -gt 0 ]] @@ -33,6 +53,10 @@ do report="$2" shift 2 ;; + -t|--template) + template="$2" + shift 2 + ;; -d|--output-dir) output_dir="$2" shift 2 @@ -68,20 +92,87 @@ then exit 1 fi -tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$report" --output "$output" $dry_run +if [[ -z "$output" ]] +then + output="$output_dir/REPLICATION-PartB-SIVACOR.md" +fi -if [[ "$replace_report" -eq 1 ]] +if [[ ! -f "$template" ]] then - if [[ ! -f "$output" ]] + if [[ -f "$report" ]] then - echo "ERROR: Generated SIVACOR Part B '$output' not found." + mkdir -p "$output_dir" + template="$output_dir/REPLICATION-PartB-template.md" + "$PYTHON" tools/create_sivacor_partb_template.py --input "$report" --output "$template" + elif [[ -f "template/original-REPLICATION.md" || -f "REPLICATION.md" ]] + then + mkdir -p "$output_dir" + source_report="REPLICATION.md" + if [[ -f "template/original-REPLICATION.md" ]] + then + source_report="template/original-REPLICATION.md" + fi + template="$output_dir/REPLICATION-PartB-template.md" + splitline=$(grep -n "You are starting \*PartB\*." "$source_report" | cut -f1 -d: | head -1 || true) + if [[ -z "$splitline" ]] + then + echo "ERROR: Could not find the Part B split marker in '$source_report'." + exit 1 + fi + raw_template="$output_dir/REPLICATION-PartB-template-raw.md" + tail -n +"$splitline" "$source_report" > "$raw_template" + "$PYTHON" tools/create_sivacor_partb_template.py --input "$raw_template" --output "$template" + else + echo "ERROR: Could not find a SIVACOR Part B template, '$report', template/original-REPLICATION.md, or REPLICATION.md." exit 1 fi +fi + +PYTHON="$PYTHON" tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$template" --output "$output" $dry_run + +if [[ "$replace_report" -eq 1 ]] +then if [[ "$dry_run" == "--dry-run" ]] then - echo "DRY RUN: Would copy '$output' to '$report'." + if [[ -f "$report" ]] + then + echo "DRY RUN: Would copy '$output' to '$report'." + elif [[ -f "REPLICATION.md" ]] + then + echo "DRY RUN: Would replace the Part B section of REPLICATION.md with '$output'." + else + echo "DRY RUN: Would copy '$output' to '$report'." + fi else - cp "$output" "$report" - echo "Copied '$output' to '$report'." + if [[ ! -f "$output" ]] + then + echo "ERROR: Generated SIVACOR Part B '$output' not found." + exit 1 + fi + if [[ -f "$report" ]] + then + cp "$output" "$report" + echo "Copied '$output' to '$report'." + elif [[ -f "REPLICATION.md" ]] + then + splitline=$(grep -n "You are starting \*PartB\*." REPLICATION.md | cut -f1 -d: | head -1 || true) + if [[ -z "$splitline" ]] + then + splitline=$(grep -n "^## All data files provided$" REPLICATION.md | cut -f1 -d: | head -1 || true) + fi + if [[ -z "$splitline" ]] + then + echo "ERROR: Could not find the Part B split marker in REPLICATION.md." + exit 1 + fi + tmp_report=$(mktemp) + head -n $(( splitline - 1 )) REPLICATION.md > "$tmp_report" + cat "$output" >> "$tmp_report" + mv "$tmp_report" REPLICATION.md + echo "Replaced the Part B section of REPLICATION.md with '$output'." + else + cp "$output" "$report" + echo "Copied '$output' to '$report'." + fi fi fi diff --git a/automations/24_amend_report.sh b/automations/24_amend_report.sh index 4417191..c8f0959 100755 --- a/automations/24_amend_report.sh +++ b/automations/24_amend_report.sh @@ -1,6 +1,25 @@ #!/bin/bash #set -ev +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi + [[ "$SkipProcessing" == "yes" ]] && exit 0 [[ -z $1 ]] && indir=generated || indir=$@ @@ -97,18 +116,26 @@ if [ ! -f "$indir/manifest.restricted.txt" ]; then echo "not present" > "$indir/manifest.restricted.txt" fi +if [ ! -f "$indir/sivacor-partb.md" ]; then + : > "$indir/sivacor-partb.md" +fi + +if [ ! -f "$indir/sivacor-partb-appendix.md" ]; then + : > "$indir/sivacor-partb-appendix.md" +fi + # Now use the template to fill in the main part tmpmain=$(mktemp) tmpapp=$(mktemp) -python3 tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain +"$PYTHON" tools/replace_placeholders.py --infile ${basefile} --indir "$indir" --outfile $tmpmain # If the {{ large-file-report.md }} was not generated, remove the placeholder -if [ ! -f "generated/large-file-report.md" ]; then - sed -i 's/{{ large-file-report.md }}/\n/' $tmpmain +if [ ! -f "$indir/large-file-report.md" ]; then + perl -0pi -e 's/\{\{ large-file-report\.md \}\}/\n/g' "$tmpmain" fi # If there is a line with "Automatically Generated Appendices", we remove it and everything after it. @@ -122,7 +149,7 @@ fi # Fill in the appendix -python3 tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix +"$PYTHON" tools/replace_placeholders.py --infile ${template_app} --indir "$indir" --outfile $appendix # DISABLED: Append the generated appendix to the base file echo "" >> $tmpapp diff --git a/docs/96-91-pipeline-overview.md b/docs/96-91-pipeline-overview.md index cf343db..c14aa1b 100644 --- a/docs/96-91-pipeline-overview.md +++ b/docs/96-91-pipeline-overview.md @@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file -- Generates `REPLICATION-PartB-SIVACOR.md` -- Preserves the normal `REPLICATION-PartB.md` template structure +- Generates `generated/REPLICATION-PartB-SIVACOR.md` +- Preserves the normal Part B template structure from `template/REPLICATION-PartB-sivacor.md` - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings -- Adds the full SIVACOR arrangement comparison to the Appendix -- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report` +- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison +- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report - Does not rerun author code - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification diff --git a/docs/automations/index.md b/docs/automations/index.md index cf343db..c14aa1b 100644 --- a/docs/automations/index.md +++ b/docs/automations/index.md @@ -102,13 +102,13 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file -- Generates `REPLICATION-PartB-SIVACOR.md` -- Preserves the normal `REPLICATION-PartB.md` template structure +- Generates `generated/REPLICATION-PartB-SIVACOR.md` +- Preserves the normal Part B template structure from `template/REPLICATION-PartB-sivacor.md` - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings -- Adds the full SIVACOR arrangement comparison to the Appendix -- Can copy the generated file over `REPLICATION-PartB.md` with `--replace-report` +- Writes `generated/sivacor-partb-appendix.md` so the normal generated appendix can include the full SIVACOR arrangement comparison +- With `--replace-report`, can copy the generated file over `REPLICATION-PartB.md` or replace the Part B section inside a single `REPLICATION.md` revision report - Does not rerun author code - Does not compare figures or tables against the manuscript, evaluate substantive code behavior, determine README requirements completeness, or assign final classification diff --git a/docs/tools/index.md b/docs/tools/index.md index 1722914..c049ddf 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -53,7 +53,7 @@ Python script for extracting computing environment, timing information, and SIVA **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/get_sivacor_info.py) | [Help](repository/96-90-get_sivacor_info.md) ### generate_sivacor_partb.sh -Bash wrapper for generating `REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file preserves the normal Part B template structure and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. +Bash wrapper for generating `generated/REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file preserves the normal Part B template structure and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. It also writes `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/generate_sivacor_partb.sh) | [Help](repository/96-90-get_sivacor_info.md) diff --git a/docs/tools/repository/96-90-get_sivacor_info.md b/docs/tools/repository/96-90-get_sivacor_info.md index dc89d93..111e516 100644 --- a/docs/tools/repository/96-90-get_sivacor_info.md +++ b/docs/tools/repository/96-90-get_sivacor_info.md @@ -27,10 +27,10 @@ python3.12 tools/get_sivacor_info.py --jobid --key --report < python3.12 tools/get_sivacor_info.py --jobid --key --report --dry-run # Generate SIVACOR Part B insert snippets from the TRO -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/partb-SIVACOR-computing-environment.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/partb-SIVACOR-replication-steps.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/partb-SIVACOR-findings.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/partb-SIVACOR-appendix.md +python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/sivacor-partb-computing-environment.md +python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/sivacor-partb-replication-steps.md +python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/sivacor-partb-findings.md +python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/sivacor-partb-appendix.md # Generate a template-consistent SIVACOR Part B file tools/generate_sivacor_partb.sh --dry-run @@ -114,7 +114,7 @@ Generates the full SIVACOR arrangement comparison for the Appendix: - Lists other generated output, data/intermediate, log, R environment, and uncategorized paths - Lists removed and modified paths, if any -`tools/generate_sivacor_partb.sh` combines these snippets with the existing `REPLICATION-PartB.md` template and writes `REPLICATION-PartB-SIVACOR.md`. `automations/18_summarize_sivacor.sh --replace-report` then copies that generated file over `REPLICATION-PartB.md`. +`tools/generate_sivacor_partb.sh` combines these snippets with `template/REPLICATION-PartB-sivacor.md` and writes `generated/REPLICATION-PartB-SIVACOR.md`. The SIVACOR appendix snippet is written to `generated/sivacor-partb-appendix.md` and included when the normal appendix template is regenerated. `automations/18_summarize_sivacor.sh --replace-report` then copies the generated Part B file over `REPLICATION-PartB.md`, or replaces the Part B section inside `REPLICATION.md` when a revision report is not split. ### Examples @@ -140,7 +140,7 @@ python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key comp # Add timing info to report python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md -# Generate REPLICATION-PartB-SIVACOR.md, then copy it over REPLICATION-PartB.md +# Generate generated/REPLICATION-PartB-SIVACOR.md, then apply it to the current report ./automations/18_summarize_sivacor.sh --replace-report # Using positional arguments @@ -153,7 +153,7 @@ python3.12 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3 ## SIVACOR Workflow Note -For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `REPLICATION-PartB-SIVACOR.md`, then copy it over `REPLICATION-PartB.md` when ready. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," a concise SIVACOR-generated file summary into "Findings," and the full arrangement comparison into the Appendix. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification. +For repositories generated by SIVACOR, do not rerun the author code as part of the AEA workflow. The submitted repository should include a `tro/` directory containing the TRO JSON-LD file. Use `tools/generate_sivacor_partb.sh` or `automations/18_summarize_sivacor.sh` to generate a template-consistent `generated/REPLICATION-PartB-SIVACOR.md`, then apply it with `--replace-report` when ready. In split-report cases, this updates `REPLICATION-PartB.md`; in single-file revision reports, it replaces the Part B section inside `REPLICATION.md`. The generated file inserts SIVACOR computing environment facts into "Computing Environment of the Replicator," SIVACOR workflow steps into "Replication steps," and a concise SIVACOR-generated file summary into "Findings." The full arrangement comparison is written to `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. Human review still compares output files against the manuscript, evaluates substantive code behavior, checks requirements completeness against the README, and assigns the final classification. ## Requirements diff --git a/template/REPLICATION-PartB-sivacor.md b/template/REPLICATION-PartB-sivacor.md new file mode 100644 index 0000000..244883f --- /dev/null +++ b/template/REPLICATION-PartB-sivacor.md @@ -0,0 +1,257 @@ +> INSTRUCTIONS: ==> Workflow stage: You are starting *PartB*. Move the *PartB* sub-task to *In Progress*. + + +## All data files provided + +The full list of data files is listed in the Appendix. + +> INSTRUCTIONS: Please verify that the list in the appendix was created and is complete. If not, create the list by hand (instructions in the appendix) + + +### File checks + +{{ file-paths-summary.md }} + +{{ duplicate-files-report.md }} + +{{ zero-byte-files-report.md }} + +{{ large-file-report.md }} + +### PII Checks + + +{{ pii-summary.md }} + + +> [NOTE] As a reminder, no data with PII that **needs to remain private** should be published in the main replication package. If you have such PII in your dataset, and are allowed to publish it with restrictions, please contact us for the best solution, but do not include it in the main replication package. If you are allowed to publish the PII data as-is, please clearly document or explain that permission, to avoid future misunderstandings. + +We recommend using the J-PAL maintained [PII-Scan for R](https://github.com/J-PAL/PII-Scan) or [PII-Scan for Stata](https://github.com/J-PAL/stata_PII_scan) to get an idea of potential PII in your dataset. + +> ⚠️ 📢 You remain ultimately responsible for ensuring that no **unauthorized** PII is included in the published replication package. + + +## Stated Requirements + +> INSTRUCTIONS: The authors may have specified specific requirements in terms of software, computer hardware, etc. Please list them here. This is **different** from the Computing Environment of the Replicator. You have the option to amend these with unstated requirements later. If all requirements are listed, check the box "Requirements are complete". + +- [ ] No requirements specified +- [ ] Operating system used: + - [ ] Windows 10/11 + - [ ] MacOS + - [ ] Linux + - [ ] Windows Server + - [ ] Not specified +- [ ] Software Requirements specified as follows: + - Software 1 + - Software 2 +- [ ] Computational Requirements specified as follows: + - Memory (RAM), number of nodes in a cluster, use of parallel processing, disk space, etc. +- [ ] Time Requirements specified as follows: + - Length of necessary computation (hours, weeks, etc.) + +- [ ] Requirements are complete. + +> INSTRUCTIONS: If the requirements are NOT complete, please leave this line in. If UNSURE, leave this line in: + +For missing requirements, see the list of required changes in the **[FINDINGS](#findings)** section. + +> INSTRUCTIONS: If easier, simply copy-and-paste the authors' stated requirements here: + +--- + +## Code description + +> INSTRUCTIONS: Review the code (but do not run it yet). Identify programs that create "analysis files" ("data preparation code"). Identify programs that create tables and figures. Not every deposit will have separate programs for this. + +> INSTRUCTIONS: Identify all **Figure, Table, and any in-text numbers**. Create a list, mapping each of them to a particular program and line number within the program (use [this template](code-check.xlsx)). Commit that list. You will come back to the list in your findings. IN THIS SECTION, point out only a summary description, including of shortcomings. E.g. + +> INSTRUCTIONS: For example, you could write "There are four provided Stata do files, three Matlab .m files, including a "master.do"." +> INSTRUCTIONS: And you could list the issues you encounter: +> INSTRUCTIONS: - Table 5: could not identify code that produces Table 5 +> INSTRUCTIONS: - Neither the program codes, nor the README, identify which tables are produced by what program. + +{{ programs-summary.txt }} + +The full list of programs provided can be found in the Appendix. + +> INSTRUCTIONS: Verify the appendix, the repository. If the list is missing, generate it by hand (instructions in the appendix). + +- [ ] The replication package contains a "main" or "master" file(s) which calls all other auxiliary programs. + +> INSTRUCTIONS: If the above checkbox for "main" file is NOT checked, leave the following SUGGESTION in the report! + +> [SUGGESTED] We strongly advise the use of a single (or a small number of) main control file(s) to automatically reproduce all figures and tables in the paper, without manual interaction. + +> NOTE: In-text numbers that reference numbers in tables do not need to be listed. Only in-text numbers that correspond to no table or figure need to be listed. + + +### Experimental/Survey instructions + +> INSTRUCTIONS: This section is only relevant if the authors conducted a survey themselves, or ran an experiment themselves. If not, please delete this entire section (including the section title). + +> INSTRUCTIONS: Check if the deposit contains the software/scripts to implement the experiment/survey. The README should point you at such steps. Sometimes, there may be an appendix that describes the survey. That is sufficient IF it also describes the actual computer files. Do check the manifest (`generated/manifest.txt`) for any such files. Qualtrics files end with `qsf`, other survey or experiment files may be specific types of Excel files, or Python code. Primarily, this should be OBVIOUS from the README. + +> INSTRUCTIONS: We do not attempt to run such code or scripts ourselves, we only ensure that they are present. + +> INSTRUCTIONS: If none are present, then leave the following text in the report. + +The deposit does not seem to contain the required software/scripts to implement the experiment/survey, though the appendix provides a complete verbose description thereof. As per the AEA's [Policy for Papers Conducting Experiments and Collecting Primary Data](https://www.aeaweb.org/journals/data/policy-experimental), please + +> [REQUIRED] Provide any computer programs, configuration files, or scripts used to run the experiment or develop the survey instrument, e.g., z-Tree code, Qualtrics, SurveyCTO, and LimeSurvey. + +> INSTRUCTIONS: If such files ARE present, then leave the following text in the report, and describe the files that are present. + +The deposit contains the following software/scripts to implement the experiment/survey: + +``` +List the files here, with a brief description of each. +``` + +## Computing Environment of the Replicator + +{{ sivacor-partb-computing-environment.md }} + +## Replication steps + +{{ sivacor-partb-replication-steps.md }} + +> INSTRUCTIONS: provide details about your process of accessing the code and data. +> +> - Do NOT detail things like "I save them on my Desktop". +> - DO describe actions that you did as per instructions ("I added a config.do") +> - DO describe any other actions you needed to do ("I had to make changes in multiple programs"), without going into TOO much detail. (Link to the log file in the JIRA comments of this case!) +> +> BUT: +> +> - DO provide ENOUGH detail so that an author, without access to the logs, can understand what needed to be fixed, including a copy-paste of the error message. +> - DO commit to git before EACH new run with corrected code. +> - DO (after all debugging is completed) a full run through the data, top-to-bottom, once all bugs are fixed, using the approriate method (command line or right-click). + +> INSTRUCTIONS: ==> Workflow stage: You are now going to *Writing Report*. Verify that both PartA and PartB have been completed. + +## Findings + +> INSTRUCTIONS: Describe your findings both positive and negative in some detail, for each **Data Preparation Code, Figure, Table, and any in-text numbers**. You can re-use the Excel file created under *Code Description*. When errors happen, be as precise as possible. For differences in figures, provide both a screenshot of what the manuscript contains, as well as the figure produced by the code you ran. For differences in numbers, provide both the number as reported in the manuscript, as well as the number replicated. If too many numbers, contact your supervisor. + +> INSTRUCTIONS: Even when there is an external reproducibility report, summarize the findings here. + +{{ sivacor-partb-findings.md }} + +### Missing Requirements + +> INSTRUCTIONS: If the replication package contains Stata programs run `tools/Stata_scan_code/scan_packages.do`, ensuring that you update the global `codedir` first. If the data is accessible, add any packages not mentioned in the README to the `config.do` and paste the excel output as a table below. If the data is restricted-access and not obtainable in a reasonable amount of time, paste the excel output as a table below. + +> INSTRUCTIONS: If it turns out that some requirements were not stated/ are incomplete (software, packages, operating system), please list (check) the *missing* list of requirements here. Remove lines that are not necessary. If the stated requirements are complete, delete this entire section, including the [REQUIRED] tag at the end, and replace with "None" + +- [ ] Software Requirements + - [ ] Stata + - [ ] Version + - Packages go here + - [ ] Matlab + - [ ] Version + - [ ] R + - [ ] Version + - R packages go here + - [ ] Python + - [ ] Version + - Python package go here + - [ ] REPLACE ME WITH OTHER +- [ ] Computational Requirements specified as follows: + - Cluster size, disk size, memory size, etc. +- [ ] Time Requirements + - Length of necessary computation (hours, weeks, etc.) + +> [REQUIRED] Please amend README to contain complete requirements. + +You can copy the section above, amended if necessary. + + +### Data Preparation Code + +Examples: + +- Program `1-create-data.do` ran without error, output expected data +- Program `2-create-appendix-data.do` failed to produce any output. + +### Tables and Figures + +> INSTRUCTIONS: Insert the filled-out `code-check.xlsx` here (complete the column `Reproduced?`), using the VS Code Plugins [Excel to Markdown table](https://marketplace.visualstudio.com/items?itemName=csholmq.excel-to-markdown-table). Then describe in more detail the issues that may have arisen. + +Examples: + +- Table 1: Looks the same +- Table 2: (contains no data) +- Table 3: Minor differences in row 5, column 3, 0.003 instead of 0.3 + +> INSTRUCTIONS: For tables, simple comparisons can be listed out as above. More complex differences can be described by using screenshots of the original table and the reproduced table, highlighting the differences. + +> INSTRUCTIONS: Please provide a comparison with the paper when describing that figures look different. Use a screenshot for the paper, and the graph generated by the programs for the comparison. Reference the graph generated by the programs as a local file within the repository. + +Example: + +- Figure 1: Looks the same +- Figure 2: no program provided +- Figure 3: Paper version looks different from the one generated by programs: + +Paper version: +![Paper version](template/dog.jpg) + +Figure 3 generated by programs: + +![Replicated version](template/odie.jpg) + +### In-Text Numbers + +> INSTRUCTIONS: list page and line number of in-text numbers. If ambiguous, cite the surrounding text, i.e., "the rate fell to 52% of all jobs: verified". + +[ ] In-text numbers not verified. + +[ ] There are no in-text numbers, or all in-text numbers stem from tables and figures. + +[ ] There are in-text numbers, but they are not identified in the code + +- Page 21, line 5: Same + + +## Classification + +> INSTRUCTIONS: Make an assessment here. +> +> Full reproduction can include a small number of apparently insignificant changes in the numbers in the table. Full reproduction also applies when changes to the programs needed to be made, but were successfully implemented. +> +> Partial reproduction means that a significant number (>25%) of programs and/or numbers are different. +> +> Note that if some data is confidential and not available, then a partial reproduction applies. This should be noted in the Reasons. +> +> Note that when all data is confidential, it is unlikely that this exercise should have been attempted. +> +> Failure to reproduce: only a small number of programs ran successfully, or only a small number of numbers were successfully generated (<25%). This also applies when all data is restricted-access and none of the **main** tables/figures are run. + +- [ ] full reproduction +- [ ] full reproduction with minor issues +- [ ] partial reproduction (see above) +- [ ] not able to reproduce most or all of the results (reasons see above) + +### Reason for incomplete reproducibility + +> INSTRUCTIONS: mark the reasons here why full reproduciblity was not achieved, and enter this information in JIRA. When results are fully reproduced, leave this section here, and mark "None". + +- [ ] None. +- [ ] `Discrepancy in output` (either figures or numbers in tables or text differ) +- [ ] `Bugs in code` that were fixable by the replicator (but should be fixed in the final deposit) +- [ ] `Code missing`, in particular if it prevented the replicator from completing the reproducibility check + - [ ] `Data preparation code missing` should be checked if the code missing seems to be data preparation code +- [ ] `Code not functional` is more severe than a simple bug: it prevented the replicator from completing the reproducibility check +- [ ] `Software not available to replicator` may happen for a variety of reasons, but in particular (a) when the software is commercial, and the replicator does not have access to a licensed copy, or (b) the software is open-source, but a specific version required to conduct the reproducibility check is not available. +- [ ] `Insufficient time available to replicator` is applicable when (a) running the code would take weeks or more, even on the best of computers (b) the replication package is very complex, and following all (manual and scripted) steps would take too long. +- [ ] `Insufficient computing resources available to replicator` is applicable when (a) running the code might take less time if sufficient compute resources were to be brought to bear, but no such resources can be accessed in a timely fashion (b) running the code is not possible on any of the accessible computing resources, or would take more than a trivial amount of money to procure (i.e., AWS or Google Cloud). +- [ ] `Data missing` is marked when data *should* be available, but was erroneously not provided, or is not accessible via the procedures described in the replication package +- [ ] `Data not available` is marked when data requires additional access steps, for instance purchase or application procedure. +- [ ] `Missing README` is marked if there is no README to guide the replicator, or the README is not in compliance with AEA requirements + + + +> INSTRUCTIONS: ==> Workflow stage: You are now going from *Writing Report* to *Submitting Report*! + +--- diff --git a/template/REPLICATION_appendix.md b/template/REPLICATION_appendix.md index f684c6d..d1f9c77 100644 --- a/template/REPLICATION_appendix.md +++ b/template/REPLICATION_appendix.md @@ -61,6 +61,8 @@ Some references for good commenting practices: {{ manifest-comparison.md }} +{{ sivacor-partb-appendix.md }} + ## Appendix: Not-for-publication data provided (if any) ``` diff --git a/tools/create_sivacor_partb_template.py b/tools/create_sivacor_partb_template.py new file mode 100644 index 0000000..d5d689b --- /dev/null +++ b/tools/create_sivacor_partb_template.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Create a SIVACOR placeholder template from a blank Part B report.""" + +import argparse +import re +import sys + + +def read_file(path): + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + + +def write_file(path, content): + with open(path, "w", encoding="utf-8") as handle: + handle.write(content.rstrip()) + handle.write("\n") + + +def replace_section(content, heading, replacement): + pattern = rf"({re.escape(heading)}\n\n).*?(?=\n## |\Z)" + updated, count = re.subn(pattern, rf"\1{replacement.strip()}\n\n", content, count=1, flags=re.DOTALL) + if count != 1: + raise ValueError(f"Could not find section {heading!r}") + return updated + + +def insert_findings_placeholder(content): + heading = "## Findings" + pos = content.find(heading) + if pos == -1: + raise ValueError("Could not find '## Findings' section") + + first_subheading = content.find("\n### ", pos + len(heading)) + if first_subheading == -1: + raise ValueError("Could not find first Findings subsection") + + placeholder = "\n\n{{ sivacor-partb-findings.md }}\n\n" + if "{{ sivacor-partb-findings.md }}" in content: + return content + return content[:first_subheading].rstrip() + placeholder + content[first_subheading:].lstrip() + + +def create_template(content): + content = replace_section( + content, + "## Computing Environment of the Replicator", + "{{ sivacor-partb-computing-environment.md }}", + ) + content = replace_section( + content, + "## Replication steps", + "{{ sivacor-partb-replication-steps.md }}", + ) + return insert_findings_placeholder(content) + + +def main(): + parser = argparse.ArgumentParser(description="Create a SIVACOR Part B placeholder template.") + parser.add_argument("--input", required=True, help="Blank Part B report or template") + parser.add_argument("--output", required=True, help="Output SIVACOR Part B template") + args = parser.parse_args() + + try: + write_file(args.output, create_template(read_file(args.input))) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_sivacor_partb.sh b/tools/generate_sivacor_partb.sh index 4dd81d8..33d7080 100755 --- a/tools/generate_sivacor_partb.sh +++ b/tools/generate_sivacor_partb.sh @@ -3,27 +3,46 @@ set -euo pipefail +PYTHON=${PYTHON:-} +if [[ -z "$PYTHON" ]] +then + if command -v python3 >/dev/null 2>&1 + then + PYTHON=python3 + elif command -v python >/dev/null 2>&1 + then + PYTHON=python + else + echo "ERROR: Could not find python3 or python in PATH." + exit 1 + fi +elif ! command -v "$PYTHON" >/dev/null 2>&1 +then + echo "ERROR: PYTHON is set to '$PYTHON', but that command was not found." + exit 1 +fi + jsonld="" output_dir="generated" -template="REPLICATION-PartB.md" -output="REPLICATION-PartB-SIVACOR.md" +template="template/REPLICATION-PartB-sivacor.md" +output="" steps_output="" findings_output="" environment_output="" appendix_output="" -report_appendix="" +empty_partb_output="" usage() { echo "Usage: $0 [-j tro.jsonld] [-t template.md] [-o output.md] [-d output_dir] [--dry-run]" echo "" echo "If -j is omitted, the first */tro/tro-*.jsonld file is used." echo "Outputs:" - echo " generated/partb-SIVACOR-computing-environment.md" - echo " generated/partb-SIVACOR-replication-steps.md" - echo " generated/partb-SIVACOR-findings.md" - echo " generated/partb-SIVACOR-appendix.md" - echo " generated/REPLICATION_appendix.md" - echo " REPLICATION-PartB-SIVACOR.md" + echo " generated/sivacor-partb-computing-environment.md" + echo " generated/sivacor-partb-replication-steps.md" + echo " generated/sivacor-partb-findings.md" + echo " generated/sivacor-partb-appendix.md" + echo " generated/sivacor-partb.md" + echo " generated/REPLICATION-PartB-SIVACOR.md" } dry_run="" @@ -74,39 +93,43 @@ then exit 1 fi +if [[ -z "$output" ]] +then + output="$output_dir/REPLICATION-PartB-SIVACOR.md" +fi + if [[ ! -f "$template" ]] then echo "ERROR: Template file '$template' not found." exit 1 fi -steps_output="$output_dir/partb-SIVACOR-replication-steps.md" -findings_output="$output_dir/partb-SIVACOR-findings.md" -environment_output="$output_dir/partb-SIVACOR-computing-environment.md" -appendix_output="$output_dir/partb-SIVACOR-appendix.md" -report_appendix="$output_dir/REPLICATION_appendix.md" +steps_output="$output_dir/sivacor-partb-replication-steps.md" +findings_output="$output_dir/sivacor-partb-findings.md" +environment_output="$output_dir/sivacor-partb-computing-environment.md" +appendix_output="$output_dir/sivacor-partb-appendix.md" +empty_partb_output="$output_dir/sivacor-partb.md" if [[ "$dry_run" != "--dry-run" ]] then mkdir -p "$output_dir" + : > "$empty_partb_output" fi -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-computing-environment --output "$environment_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-replication-steps --output "$steps_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-findings --output "$findings_output" $dry_run -python3 tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-appendix --output "$appendix_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-computing-environment --output "$environment_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-replication-steps --output "$steps_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-findings --output "$findings_output" $dry_run +"$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-appendix --output "$appendix_output" $dry_run if [[ "$dry_run" == "--dry-run" ]] then if [[ -f "$environment_output" && -f "$steps_output" && -f "$findings_output" && -f "$appendix_output" ]] then - python3 tools/insert_sivacor_partb_sections.py --template "$template" --computing-environment "$environment_output" --replication-steps "$steps_output" --findings "$findings_output" --output "$output" --dry-run - python3 tools/update_sivacor_appendix.py --appendix "$report_appendix" --sivacor-appendix "$appendix_output" --dry-run + "$PYTHON" tools/replace_placeholders.py --infile "$template" --indir "$output_dir" --outfile /dev/stdout else echo "DRY RUN: Would combine generated snippets with '$template' into '$output'." - echo "DRY RUN: Would add '$appendix_output' to '$report_appendix'." + echo "DRY RUN: Would write the SIVACOR appendix snippet to '$appendix_output'." fi else - python3 tools/insert_sivacor_partb_sections.py --template "$template" --computing-environment "$environment_output" --replication-steps "$steps_output" --findings "$findings_output" --output "$output" - python3 tools/update_sivacor_appendix.py --appendix "$report_appendix" --sivacor-appendix "$appendix_output" + "$PYTHON" tools/replace_placeholders.py --infile "$template" --indir "$output_dir" --outfile "$output" fi diff --git a/tools/get_sivacor_info.py b/tools/get_sivacor_info.py index 6fc7fa8..56da287 100755 --- a/tools/get_sivacor_info.py +++ b/tools/get_sivacor_info.py @@ -1171,7 +1171,7 @@ def main(): %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key computing %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key time %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb - %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb-sivacor --output REPLICATION-PartB-SIVACOR.md + %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key partb-sivacor --output generated/REPLICATION-PartB-SIVACOR.md %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md %(prog)s --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md --dry-run """ diff --git a/tools/insert_sivacor_partb_sections.py b/tools/insert_sivacor_partb_sections.py deleted file mode 100644 index ff9176a..0000000 --- a/tools/insert_sivacor_partb_sections.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -"""Insert SIVACOR sections into an existing Part B Markdown template.""" - -import argparse -import re -import sys - - -def read_file(path): - with open(path, "r", encoding="utf-8") as handle: - return handle.read() - - -def write_file(path, content): - with open(path, "w", encoding="utf-8") as handle: - handle.write(content.rstrip()) - handle.write("\n") - - -def insert_after_replication_steps(content, insert_text): - marker = "## Replication steps" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Replication steps' section") - - next_heading = content.find("\n## ", pos + len(marker)) - section_end = next_heading if next_heading != -1 else len(content) - replacement = marker + "\n\n" + insert_text.strip() + "\n\n" - return content[:pos] + replacement + content[section_end:].lstrip() - - -def replace_computing_environment(content, insert_text): - marker = "## Computing Environment of the Replicator" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Computing Environment of the Replicator' section") - - next_heading = content.find("\n## ", pos + len(marker)) - section_end = next_heading if next_heading != -1 else len(content) - replacement = marker + "\n\n" + insert_text.strip() + "\n\n" - return content[:pos] + replacement + content[section_end:].lstrip() - - -def insert_after_findings_intro(content, insert_text): - marker = "## Findings" - pos = content.find(marker) - if pos == -1: - raise ValueError("Could not find '## Findings' section") - - first_subheading = content.find("\n### ", pos + len(marker)) - if first_subheading == -1: - raise ValueError("Could not find first Findings subsection") - - return content[:first_subheading].rstrip() + "\n\n" + insert_text + "\n\n" + content[first_subheading:].lstrip() - - -def remove_existing_sivacor_sections(content): - content = re.sub( - r"\nThe AEA reviewer did not rerun the author code\. The SIVACOR TRO records the following execution environment:\n\n.*?(?=\n## Replication steps)", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n\*\*SIVACOR automated execution summary\*\*.*?(?=\n- \[ \] The reproducibility check|\n> INSTRUCTIONS: provide details|\n## Findings)", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n### SIVACOR-generated files\n\n.*?(?=\n### |\n## )", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n## Appendix\n\n### SIVACOR arrangement comparison\n\n.*\Z", - "\n", - content, - flags=re.DOTALL, - ) - content = re.sub( - r"\n## Appendix: SIVACOR arrangement comparison\n\n.*\Z", - "\n", - content, - flags=re.DOTALL, - ) - return content - - -def main(): - parser = argparse.ArgumentParser(description="Insert generated SIVACOR Part B sections into a Markdown template.") - parser.add_argument("--template", required=True, help="Existing Part B Markdown template") - parser.add_argument("--computing-environment", required=True, help="Generated SIVACOR computing environment Markdown") - parser.add_argument("--replication-steps", required=True, help="Generated SIVACOR replication steps Markdown") - parser.add_argument("--findings", required=True, help="Generated SIVACOR findings Markdown") - parser.add_argument("--output", required=True, help="Output Markdown file") - parser.add_argument("--dry-run", action="store_true", help="Print output instead of writing") - args = parser.parse_args() - - content = remove_existing_sivacor_sections(read_file(args.template)) - content = replace_computing_environment(content, read_file(args.computing_environment).strip()) - content = insert_after_replication_steps(content, read_file(args.replication_steps).strip()) - content = insert_after_findings_intro(content, read_file(args.findings).strip()) - - if args.dry_run: - print(content) - else: - write_file(args.output, content) - print(f"Successfully wrote {args.output}.", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/tools/replace_placeholders.py b/tools/replace_placeholders.py index 12e3750..35c65ac 100755 --- a/tools/replace_placeholders.py +++ b/tools/replace_placeholders.py @@ -2,7 +2,11 @@ import os import shutil import argparse -import chardet + +try: + import chardet +except ImportError: + chardet = None TEMPLATE='REPLICATION.md' @@ -34,9 +38,13 @@ def replace_content(template,replacement,tag): # Detect encoding automatically with open(filepath, 'rb') as f: rawdata = f.read() - detected = chardet.detect(rawdata) - encoding = detected['encoding'] or 'utf-8' # Default to utf-8 if detection fails - confidence = detected['confidence'] + if chardet: + detected = chardet.detect(rawdata) + encoding = detected['encoding'] or 'utf-8' # Default to utf-8 if detection fails + confidence = detected['confidence'] + else: + encoding = 'utf-8' + confidence = 1.0 # Warn if encoding is not UTF-8 or confidence is low if encoding.lower() not in ['utf-8', 'utf-8-sig', 'ascii']: diff --git a/tools/update_sivacor_appendix.py b/tools/update_sivacor_appendix.py deleted file mode 100644 index 1af770b..0000000 --- a/tools/update_sivacor_appendix.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -"""Insert SIVACOR appendix material into generated/REPLICATION_appendix.md.""" - -import argparse -import os -import re -import sys - - -SECTION_PATTERN = re.compile( - r"\n## Appendix: SIVACOR arrangement comparison\n\n.*?(?=\n## Appendix: |\Z)", - flags=re.DOTALL, -) - - -def read_file(path): - with open(path, "r", encoding="utf-8") as handle: - return handle.read() - - -def write_file(path, content): - with open(path, "w", encoding="utf-8") as handle: - handle.write(content) - - -def normalize_heading(content): - return re.sub(r"\A### SIVACOR arrangement comparison", "## Appendix: SIVACOR arrangement comparison", content.strip()) - - -def update_appendix(existing_content, sivacor_content): - section = normalize_heading(sivacor_content) - if existing_content.strip(): - base = SECTION_PATTERN.sub("", existing_content).rstrip() - return base + "\n\n" + section + "\n" - return "# Automatically Generated Appendices\n\n> No action is required, unless indicated in the main body of the report.\n\n" + section + "\n" - - -def main(): - parser = argparse.ArgumentParser(description="Add SIVACOR arrangement comparison to generated appendix.") - parser.add_argument("--appendix", default="generated/REPLICATION_appendix.md", help="Generated appendix file to update") - parser.add_argument("--sivacor-appendix", required=True, help="Generated SIVACOR appendix snippet") - parser.add_argument("--dry-run", action="store_true", help="Print output instead of writing") - args = parser.parse_args() - - sivacor_content = read_file(args.sivacor_appendix) - existing_content = read_file(args.appendix) if os.path.exists(args.appendix) else "" - updated = update_appendix(existing_content, sivacor_content) - - if args.dry_run: - print(updated) - else: - os.makedirs(os.path.dirname(args.appendix) or ".", exist_ok=True) - write_file(args.appendix, updated) - print(f"Successfully updated {args.appendix}.", file=sys.stderr) - - -if __name__ == "__main__": - main() From dbfb13ffab77163a9e6e7e9cec9bd1f785f5626e Mon Sep 17 00:00:00 2001 From: dmcguir5 Date: Mon, 22 Jun 2026 11:24:47 -0400 Subject: [PATCH 2/3] Address SIVACOR review feedback from June 18 --- REPLICATION.md | 5 + automations/18_summarize_sivacor.sh | 31 +-- automations/24_amend_report.sh | 19 +- docs/96-91-pipeline-overview.md | 2 +- docs/automations/index.md | 2 +- docs/tools/index.md | 2 +- .../repository/96-90-get_sivacor_info.md | 34 +-- template/REPLICATION-PartB-sivacor.md | 257 ------------------ tools/create_sivacor_partb_template.py | 72 ----- tools/generate_sivacor_partb.sh | 29 +- tools/get_sivacor_info.py | 2 +- 11 files changed, 61 insertions(+), 394 deletions(-) delete mode 100644 template/REPLICATION-PartB-sivacor.md delete mode 100644 tools/create_sivacor_partb_template.py diff --git a/REPLICATION.md b/REPLICATION.md index 4117a50..ddcaf7c 100644 --- a/REPLICATION.md +++ b/REPLICATION.md @@ -411,6 +411,8 @@ List the files here, with a brief description of each. - MATLAB R2025b - Intel Compiler 3.14152 (note: there is no such thing, so please verify the version!) +{{ sivacor-partb-computing-environment.md }} + ## Replication steps @@ -444,6 +446,8 @@ error: command distinct unknown - Code failed because of a typo in the name of the file "`superdata.dta`" (was: `superdta.dta`). Fixed. Ran again. - Code ran to completion. +{{ sivacor-partb-replication-steps.md }} + > INSTRUCTIONS: ==> Workflow stage: You are now going to *Writing Report*. Verify that both PartA and PartB have been completed. ## Findings @@ -452,6 +456,7 @@ error: command distinct unknown > INSTRUCTIONS: Even when there is an external reproducibility report, summarize the findings here. +{{ sivacor-partb-findings.md }} ### Missing Requirements diff --git a/automations/18_summarize_sivacor.sh b/automations/18_summarize_sivacor.sh index 8c6f5dd..d7cffd7 100755 --- a/automations/18_summarize_sivacor.sh +++ b/automations/18_summarize_sivacor.sh @@ -27,7 +27,7 @@ jsonld="" output_dir="generated" output="" report="REPLICATION-PartB.md" -template="template/REPLICATION-PartB-sivacor.md" +template="REPLICATION.md" dry_run="" replace_report=0 @@ -99,33 +99,8 @@ fi if [[ ! -f "$template" ]] then - if [[ -f "$report" ]] - then - mkdir -p "$output_dir" - template="$output_dir/REPLICATION-PartB-template.md" - "$PYTHON" tools/create_sivacor_partb_template.py --input "$report" --output "$template" - elif [[ -f "template/original-REPLICATION.md" || -f "REPLICATION.md" ]] - then - mkdir -p "$output_dir" - source_report="REPLICATION.md" - if [[ -f "template/original-REPLICATION.md" ]] - then - source_report="template/original-REPLICATION.md" - fi - template="$output_dir/REPLICATION-PartB-template.md" - splitline=$(grep -n "You are starting \*PartB\*." "$source_report" | cut -f1 -d: | head -1 || true) - if [[ -z "$splitline" ]] - then - echo "ERROR: Could not find the Part B split marker in '$source_report'." - exit 1 - fi - raw_template="$output_dir/REPLICATION-PartB-template-raw.md" - tail -n +"$splitline" "$source_report" > "$raw_template" - "$PYTHON" tools/create_sivacor_partb_template.py --input "$raw_template" --output "$template" - else - echo "ERROR: Could not find a SIVACOR Part B template, '$report', template/original-REPLICATION.md, or REPLICATION.md." - exit 1 - fi + echo "ERROR: Template file '$template' not found." + exit 1 fi PYTHON="$PYTHON" tools/generate_sivacor_partb.sh --jsonld "$jsonld" --output-dir "$output_dir" --template "$template" --output "$output" $dry_run diff --git a/automations/24_amend_report.sh b/automations/24_amend_report.sh index c8f0959..81bb373 100755 --- a/automations/24_amend_report.sh +++ b/automations/24_amend_report.sh @@ -116,13 +116,16 @@ if [ ! -f "$indir/manifest.restricted.txt" ]; then echo "not present" > "$indir/manifest.restricted.txt" fi -if [ ! -f "$indir/sivacor-partb.md" ]; then - : > "$indir/sivacor-partb.md" -fi - -if [ ! -f "$indir/sivacor-partb-appendix.md" ]; then - : > "$indir/sivacor-partb-appendix.md" -fi +for sivacor_file in \ + sivacor-partb-computing-environment.md \ + sivacor-partb-replication-steps.md \ + sivacor-partb-findings.md \ + sivacor-partb-appendix.md +do + if [ ! -f "$indir/$sivacor_file" ]; then + : > "$indir/$sivacor_file" + fi +done @@ -135,7 +138,7 @@ tmpapp=$(mktemp) # If the {{ large-file-report.md }} was not generated, remove the placeholder if [ ! -f "$indir/large-file-report.md" ]; then - perl -0pi -e 's/\{\{ large-file-report\.md \}\}/\n/g' "$tmpmain" + "$PYTHON" -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text(encoding="utf-8").replace("{{ large-file-report.md }}", "\n"), encoding="utf-8")' "$tmpmain" fi # If there is a line with "Automatically Generated Appendices", we remove it and everything after it. diff --git a/docs/96-91-pipeline-overview.md b/docs/96-91-pipeline-overview.md index c14aa1b..0a939db 100644 --- a/docs/96-91-pipeline-overview.md +++ b/docs/96-91-pipeline-overview.md @@ -103,7 +103,7 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file - Generates `generated/REPLICATION-PartB-SIVACOR.md` -- Preserves the normal Part B template structure from `template/REPLICATION-PartB-sivacor.md` +- Extracts Part B from the single `REPLICATION.md` template during preprocessing - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings diff --git a/docs/automations/index.md b/docs/automations/index.md index c14aa1b..0a939db 100644 --- a/docs/automations/index.md +++ b/docs/automations/index.md @@ -103,7 +103,7 @@ Same as standard pipeline, but optimized for large deposits **What it does**: - Finds the submitted TRO JSON-LD file - Generates `generated/REPLICATION-PartB-SIVACOR.md` -- Preserves the normal Part B template structure from `template/REPLICATION-PartB-sivacor.md` +- Extracts Part B from the single `REPLICATION.md` template during preprocessing - Inserts the SIVACOR execution environment into Computing Environment of the Replicator - Inserts what SIVACOR actually ran into Replication steps - Inserts a concise arrangement 0 vs highest-arrangement summary into Findings diff --git a/docs/tools/index.md b/docs/tools/index.md index c049ddf..6cb5d72 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -53,7 +53,7 @@ Python script for extracting computing environment, timing information, and SIVA **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/get_sivacor_info.py) | [Help](repository/96-90-get_sivacor_info.md) ### generate_sivacor_partb.sh -Bash wrapper for generating `generated/REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file preserves the normal Part B template structure and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. It also writes `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. +Bash wrapper for generating `generated/REPLICATION-PartB-SIVACOR.md` from a submitted SIVACOR TRO file without rerunning author code. The generated file extracts Part B from the single `REPLICATION.md` template and inserts only objective SIVACOR-derived material: computing environment facts, workflow steps, and SIVACOR-generated-file findings. It also writes `generated/sivacor-partb-appendix.md` for inclusion by the normal generated appendix template. **Links:** [Source](https://github.com/aeaDataEditor/replication-template/blob/master/tools/generate_sivacor_partb.sh) | [Help](repository/96-90-get_sivacor_info.md) diff --git a/docs/tools/repository/96-90-get_sivacor_info.md b/docs/tools/repository/96-90-get_sivacor_info.md index 111e516..2d3d1fc 100644 --- a/docs/tools/repository/96-90-get_sivacor_info.md +++ b/docs/tools/repository/96-90-get_sivacor_info.md @@ -16,21 +16,21 @@ Parses SIVACOR JSONLD (TRO - Transparent Research Object) files to extract compu ```bash # Output to stdout -python3.12 tools/get_sivacor_info.py -python3.12 tools/get_sivacor_info.py --jsonld --key -python3.12 tools/get_sivacor_info.py --jobid --key +python3 tools/get_sivacor_info.py +python3 tools/get_sivacor_info.py --jsonld --key +python3 tools/get_sivacor_info.py --jobid --key # Update report file -python3.12 tools/get_sivacor_info.py --jobid --key --report +python3 tools/get_sivacor_info.py --jobid --key --report # Dry-run (preview without updating) -python3.12 tools/get_sivacor_info.py --jobid --key --report --dry-run +python3 tools/get_sivacor_info.py --jobid --key --report --dry-run # Generate SIVACOR Part B insert snippets from the TRO -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/sivacor-partb-computing-environment.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/sivacor-partb-replication-steps.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/sivacor-partb-findings.md -python3.12 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/sivacor-partb-appendix.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-computing-environment --output generated/sivacor-partb-computing-environment.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-replication-steps --output generated/sivacor-partb-replication-steps.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-findings --output generated/sivacor-partb-findings.md +python3 tools/get_sivacor_info.py --jsonld 246665/tro/tro-6a23045802a927359ccb67f4.jsonld --key sivacor-appendix --output generated/sivacor-partb-appendix.md # Generate a template-consistent SIVACOR Part B file tools/generate_sivacor_partb.sh --dry-run @@ -114,41 +114,41 @@ Generates the full SIVACOR arrangement comparison for the Appendix: - Lists other generated output, data/intermediate, log, R environment, and uncategorized paths - Lists removed and modified paths, if any -`tools/generate_sivacor_partb.sh` combines these snippets with `template/REPLICATION-PartB-sivacor.md` and writes `generated/REPLICATION-PartB-SIVACOR.md`. The SIVACOR appendix snippet is written to `generated/sivacor-partb-appendix.md` and included when the normal appendix template is regenerated. `automations/18_summarize_sivacor.sh --replace-report` then copies the generated Part B file over `REPLICATION-PartB.md`, or replaces the Part B section inside `REPLICATION.md` when a revision report is not split. +`tools/generate_sivacor_partb.sh` extracts Part B from the single `REPLICATION.md` template, fills its SIVACOR placeholders, and writes `generated/REPLICATION-PartB-SIVACOR.md`. For non-SIVACOR reports, normal preprocessing replaces those placeholders with empty content. The SIVACOR appendix snippet is written to `generated/sivacor-partb-appendix.md` and included when the normal appendix template is regenerated. `automations/18_summarize_sivacor.sh --replace-report` then copies the generated Part B file over `REPLICATION-PartB.md`, or replaces the Part B section inside `REPLICATION.md` when a revision report is not split. ### Examples ```bash # Extract computing info and print to stdout -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing # Extract timing info and print to stdout -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time # Generate a template-consistent SIVACOR Part B file tools/generate_sivacor_partb.sh # Preview what would be added to report (dry-run) -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md --dry-run # Preview SIVACOR Part B generation ./automations/18_summarize_sivacor.sh --dry-run # Add computing info to report -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key computing --report REPLICATION-PartB.md # Add timing info to report -python3.12 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md +python3 tools/get_sivacor_info.py --jobid 69cede1db3a6af67b1c01c3d --key time --report REPLICATION-PartB.md # Generate generated/REPLICATION-PartB-SIVACOR.md, then apply it to the current report ./automations/18_summarize_sivacor.sh --replace-report # Using positional arguments cd 246302 -python3.12 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing +python3 ../tools/get_sivacor_info.py tro-69cede1db3a6af67b1c01c3d.jsonld computing # Using file path directly -python3.12 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time +python3 tools/get_sivacor_info.py --jsonld 246302/tro-69cede1db3a6af67b1c01c3d.jsonld --key time ``` ## SIVACOR Workflow Note diff --git a/template/REPLICATION-PartB-sivacor.md b/template/REPLICATION-PartB-sivacor.md deleted file mode 100644 index 244883f..0000000 --- a/template/REPLICATION-PartB-sivacor.md +++ /dev/null @@ -1,257 +0,0 @@ -> INSTRUCTIONS: ==> Workflow stage: You are starting *PartB*. Move the *PartB* sub-task to *In Progress*. - - -## All data files provided - -The full list of data files is listed in the Appendix. - -> INSTRUCTIONS: Please verify that the list in the appendix was created and is complete. If not, create the list by hand (instructions in the appendix) - - -### File checks - -{{ file-paths-summary.md }} - -{{ duplicate-files-report.md }} - -{{ zero-byte-files-report.md }} - -{{ large-file-report.md }} - -### PII Checks - - -{{ pii-summary.md }} - - -> [NOTE] As a reminder, no data with PII that **needs to remain private** should be published in the main replication package. If you have such PII in your dataset, and are allowed to publish it with restrictions, please contact us for the best solution, but do not include it in the main replication package. If you are allowed to publish the PII data as-is, please clearly document or explain that permission, to avoid future misunderstandings. - -We recommend using the J-PAL maintained [PII-Scan for R](https://github.com/J-PAL/PII-Scan) or [PII-Scan for Stata](https://github.com/J-PAL/stata_PII_scan) to get an idea of potential PII in your dataset. - -> ⚠️ 📢 You remain ultimately responsible for ensuring that no **unauthorized** PII is included in the published replication package. - - -## Stated Requirements - -> INSTRUCTIONS: The authors may have specified specific requirements in terms of software, computer hardware, etc. Please list them here. This is **different** from the Computing Environment of the Replicator. You have the option to amend these with unstated requirements later. If all requirements are listed, check the box "Requirements are complete". - -- [ ] No requirements specified -- [ ] Operating system used: - - [ ] Windows 10/11 - - [ ] MacOS - - [ ] Linux - - [ ] Windows Server - - [ ] Not specified -- [ ] Software Requirements specified as follows: - - Software 1 - - Software 2 -- [ ] Computational Requirements specified as follows: - - Memory (RAM), number of nodes in a cluster, use of parallel processing, disk space, etc. -- [ ] Time Requirements specified as follows: - - Length of necessary computation (hours, weeks, etc.) - -- [ ] Requirements are complete. - -> INSTRUCTIONS: If the requirements are NOT complete, please leave this line in. If UNSURE, leave this line in: - -For missing requirements, see the list of required changes in the **[FINDINGS](#findings)** section. - -> INSTRUCTIONS: If easier, simply copy-and-paste the authors' stated requirements here: - ---- - -## Code description - -> INSTRUCTIONS: Review the code (but do not run it yet). Identify programs that create "analysis files" ("data preparation code"). Identify programs that create tables and figures. Not every deposit will have separate programs for this. - -> INSTRUCTIONS: Identify all **Figure, Table, and any in-text numbers**. Create a list, mapping each of them to a particular program and line number within the program (use [this template](code-check.xlsx)). Commit that list. You will come back to the list in your findings. IN THIS SECTION, point out only a summary description, including of shortcomings. E.g. - -> INSTRUCTIONS: For example, you could write "There are four provided Stata do files, three Matlab .m files, including a "master.do"." -> INSTRUCTIONS: And you could list the issues you encounter: -> INSTRUCTIONS: - Table 5: could not identify code that produces Table 5 -> INSTRUCTIONS: - Neither the program codes, nor the README, identify which tables are produced by what program. - -{{ programs-summary.txt }} - -The full list of programs provided can be found in the Appendix. - -> INSTRUCTIONS: Verify the appendix, the repository. If the list is missing, generate it by hand (instructions in the appendix). - -- [ ] The replication package contains a "main" or "master" file(s) which calls all other auxiliary programs. - -> INSTRUCTIONS: If the above checkbox for "main" file is NOT checked, leave the following SUGGESTION in the report! - -> [SUGGESTED] We strongly advise the use of a single (or a small number of) main control file(s) to automatically reproduce all figures and tables in the paper, without manual interaction. - -> NOTE: In-text numbers that reference numbers in tables do not need to be listed. Only in-text numbers that correspond to no table or figure need to be listed. - - -### Experimental/Survey instructions - -> INSTRUCTIONS: This section is only relevant if the authors conducted a survey themselves, or ran an experiment themselves. If not, please delete this entire section (including the section title). - -> INSTRUCTIONS: Check if the deposit contains the software/scripts to implement the experiment/survey. The README should point you at such steps. Sometimes, there may be an appendix that describes the survey. That is sufficient IF it also describes the actual computer files. Do check the manifest (`generated/manifest.txt`) for any such files. Qualtrics files end with `qsf`, other survey or experiment files may be specific types of Excel files, or Python code. Primarily, this should be OBVIOUS from the README. - -> INSTRUCTIONS: We do not attempt to run such code or scripts ourselves, we only ensure that they are present. - -> INSTRUCTIONS: If none are present, then leave the following text in the report. - -The deposit does not seem to contain the required software/scripts to implement the experiment/survey, though the appendix provides a complete verbose description thereof. As per the AEA's [Policy for Papers Conducting Experiments and Collecting Primary Data](https://www.aeaweb.org/journals/data/policy-experimental), please - -> [REQUIRED] Provide any computer programs, configuration files, or scripts used to run the experiment or develop the survey instrument, e.g., z-Tree code, Qualtrics, SurveyCTO, and LimeSurvey. - -> INSTRUCTIONS: If such files ARE present, then leave the following text in the report, and describe the files that are present. - -The deposit contains the following software/scripts to implement the experiment/survey: - -``` -List the files here, with a brief description of each. -``` - -## Computing Environment of the Replicator - -{{ sivacor-partb-computing-environment.md }} - -## Replication steps - -{{ sivacor-partb-replication-steps.md }} - -> INSTRUCTIONS: provide details about your process of accessing the code and data. -> -> - Do NOT detail things like "I save them on my Desktop". -> - DO describe actions that you did as per instructions ("I added a config.do") -> - DO describe any other actions you needed to do ("I had to make changes in multiple programs"), without going into TOO much detail. (Link to the log file in the JIRA comments of this case!) -> -> BUT: -> -> - DO provide ENOUGH detail so that an author, without access to the logs, can understand what needed to be fixed, including a copy-paste of the error message. -> - DO commit to git before EACH new run with corrected code. -> - DO (after all debugging is completed) a full run through the data, top-to-bottom, once all bugs are fixed, using the approriate method (command line or right-click). - -> INSTRUCTIONS: ==> Workflow stage: You are now going to *Writing Report*. Verify that both PartA and PartB have been completed. - -## Findings - -> INSTRUCTIONS: Describe your findings both positive and negative in some detail, for each **Data Preparation Code, Figure, Table, and any in-text numbers**. You can re-use the Excel file created under *Code Description*. When errors happen, be as precise as possible. For differences in figures, provide both a screenshot of what the manuscript contains, as well as the figure produced by the code you ran. For differences in numbers, provide both the number as reported in the manuscript, as well as the number replicated. If too many numbers, contact your supervisor. - -> INSTRUCTIONS: Even when there is an external reproducibility report, summarize the findings here. - -{{ sivacor-partb-findings.md }} - -### Missing Requirements - -> INSTRUCTIONS: If the replication package contains Stata programs run `tools/Stata_scan_code/scan_packages.do`, ensuring that you update the global `codedir` first. If the data is accessible, add any packages not mentioned in the README to the `config.do` and paste the excel output as a table below. If the data is restricted-access and not obtainable in a reasonable amount of time, paste the excel output as a table below. - -> INSTRUCTIONS: If it turns out that some requirements were not stated/ are incomplete (software, packages, operating system), please list (check) the *missing* list of requirements here. Remove lines that are not necessary. If the stated requirements are complete, delete this entire section, including the [REQUIRED] tag at the end, and replace with "None" - -- [ ] Software Requirements - - [ ] Stata - - [ ] Version - - Packages go here - - [ ] Matlab - - [ ] Version - - [ ] R - - [ ] Version - - R packages go here - - [ ] Python - - [ ] Version - - Python package go here - - [ ] REPLACE ME WITH OTHER -- [ ] Computational Requirements specified as follows: - - Cluster size, disk size, memory size, etc. -- [ ] Time Requirements - - Length of necessary computation (hours, weeks, etc.) - -> [REQUIRED] Please amend README to contain complete requirements. - -You can copy the section above, amended if necessary. - - -### Data Preparation Code - -Examples: - -- Program `1-create-data.do` ran without error, output expected data -- Program `2-create-appendix-data.do` failed to produce any output. - -### Tables and Figures - -> INSTRUCTIONS: Insert the filled-out `code-check.xlsx` here (complete the column `Reproduced?`), using the VS Code Plugins [Excel to Markdown table](https://marketplace.visualstudio.com/items?itemName=csholmq.excel-to-markdown-table). Then describe in more detail the issues that may have arisen. - -Examples: - -- Table 1: Looks the same -- Table 2: (contains no data) -- Table 3: Minor differences in row 5, column 3, 0.003 instead of 0.3 - -> INSTRUCTIONS: For tables, simple comparisons can be listed out as above. More complex differences can be described by using screenshots of the original table and the reproduced table, highlighting the differences. - -> INSTRUCTIONS: Please provide a comparison with the paper when describing that figures look different. Use a screenshot for the paper, and the graph generated by the programs for the comparison. Reference the graph generated by the programs as a local file within the repository. - -Example: - -- Figure 1: Looks the same -- Figure 2: no program provided -- Figure 3: Paper version looks different from the one generated by programs: - -Paper version: -![Paper version](template/dog.jpg) - -Figure 3 generated by programs: - -![Replicated version](template/odie.jpg) - -### In-Text Numbers - -> INSTRUCTIONS: list page and line number of in-text numbers. If ambiguous, cite the surrounding text, i.e., "the rate fell to 52% of all jobs: verified". - -[ ] In-text numbers not verified. - -[ ] There are no in-text numbers, or all in-text numbers stem from tables and figures. - -[ ] There are in-text numbers, but they are not identified in the code - -- Page 21, line 5: Same - - -## Classification - -> INSTRUCTIONS: Make an assessment here. -> -> Full reproduction can include a small number of apparently insignificant changes in the numbers in the table. Full reproduction also applies when changes to the programs needed to be made, but were successfully implemented. -> -> Partial reproduction means that a significant number (>25%) of programs and/or numbers are different. -> -> Note that if some data is confidential and not available, then a partial reproduction applies. This should be noted in the Reasons. -> -> Note that when all data is confidential, it is unlikely that this exercise should have been attempted. -> -> Failure to reproduce: only a small number of programs ran successfully, or only a small number of numbers were successfully generated (<25%). This also applies when all data is restricted-access and none of the **main** tables/figures are run. - -- [ ] full reproduction -- [ ] full reproduction with minor issues -- [ ] partial reproduction (see above) -- [ ] not able to reproduce most or all of the results (reasons see above) - -### Reason for incomplete reproducibility - -> INSTRUCTIONS: mark the reasons here why full reproduciblity was not achieved, and enter this information in JIRA. When results are fully reproduced, leave this section here, and mark "None". - -- [ ] None. -- [ ] `Discrepancy in output` (either figures or numbers in tables or text differ) -- [ ] `Bugs in code` that were fixable by the replicator (but should be fixed in the final deposit) -- [ ] `Code missing`, in particular if it prevented the replicator from completing the reproducibility check - - [ ] `Data preparation code missing` should be checked if the code missing seems to be data preparation code -- [ ] `Code not functional` is more severe than a simple bug: it prevented the replicator from completing the reproducibility check -- [ ] `Software not available to replicator` may happen for a variety of reasons, but in particular (a) when the software is commercial, and the replicator does not have access to a licensed copy, or (b) the software is open-source, but a specific version required to conduct the reproducibility check is not available. -- [ ] `Insufficient time available to replicator` is applicable when (a) running the code would take weeks or more, even on the best of computers (b) the replication package is very complex, and following all (manual and scripted) steps would take too long. -- [ ] `Insufficient computing resources available to replicator` is applicable when (a) running the code might take less time if sufficient compute resources were to be brought to bear, but no such resources can be accessed in a timely fashion (b) running the code is not possible on any of the accessible computing resources, or would take more than a trivial amount of money to procure (i.e., AWS or Google Cloud). -- [ ] `Data missing` is marked when data *should* be available, but was erroneously not provided, or is not accessible via the procedures described in the replication package -- [ ] `Data not available` is marked when data requires additional access steps, for instance purchase or application procedure. -- [ ] `Missing README` is marked if there is no README to guide the replicator, or the README is not in compliance with AEA requirements - - - -> INSTRUCTIONS: ==> Workflow stage: You are now going from *Writing Report* to *Submitting Report*! - ---- diff --git a/tools/create_sivacor_partb_template.py b/tools/create_sivacor_partb_template.py deleted file mode 100644 index d5d689b..0000000 --- a/tools/create_sivacor_partb_template.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Create a SIVACOR placeholder template from a blank Part B report.""" - -import argparse -import re -import sys - - -def read_file(path): - with open(path, "r", encoding="utf-8") as handle: - return handle.read() - - -def write_file(path, content): - with open(path, "w", encoding="utf-8") as handle: - handle.write(content.rstrip()) - handle.write("\n") - - -def replace_section(content, heading, replacement): - pattern = rf"({re.escape(heading)}\n\n).*?(?=\n## |\Z)" - updated, count = re.subn(pattern, rf"\1{replacement.strip()}\n\n", content, count=1, flags=re.DOTALL) - if count != 1: - raise ValueError(f"Could not find section {heading!r}") - return updated - - -def insert_findings_placeholder(content): - heading = "## Findings" - pos = content.find(heading) - if pos == -1: - raise ValueError("Could not find '## Findings' section") - - first_subheading = content.find("\n### ", pos + len(heading)) - if first_subheading == -1: - raise ValueError("Could not find first Findings subsection") - - placeholder = "\n\n{{ sivacor-partb-findings.md }}\n\n" - if "{{ sivacor-partb-findings.md }}" in content: - return content - return content[:first_subheading].rstrip() + placeholder + content[first_subheading:].lstrip() - - -def create_template(content): - content = replace_section( - content, - "## Computing Environment of the Replicator", - "{{ sivacor-partb-computing-environment.md }}", - ) - content = replace_section( - content, - "## Replication steps", - "{{ sivacor-partb-replication-steps.md }}", - ) - return insert_findings_placeholder(content) - - -def main(): - parser = argparse.ArgumentParser(description="Create a SIVACOR Part B placeholder template.") - parser.add_argument("--input", required=True, help="Blank Part B report or template") - parser.add_argument("--output", required=True, help="Output SIVACOR Part B template") - args = parser.parse_args() - - try: - write_file(args.output, create_template(read_file(args.input))) - except ValueError as exc: - print(f"ERROR: {exc}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/generate_sivacor_partb.sh b/tools/generate_sivacor_partb.sh index 33d7080..6b69da2 100755 --- a/tools/generate_sivacor_partb.sh +++ b/tools/generate_sivacor_partb.sh @@ -24,13 +24,12 @@ fi jsonld="" output_dir="generated" -template="template/REPLICATION-PartB-sivacor.md" +template="REPLICATION.md" output="" steps_output="" findings_output="" environment_output="" appendix_output="" -empty_partb_output="" usage() { echo "Usage: $0 [-j tro.jsonld] [-t template.md] [-o output.md] [-d output_dir] [--dry-run]" @@ -41,7 +40,6 @@ usage() { echo " generated/sivacor-partb-replication-steps.md" echo " generated/sivacor-partb-findings.md" echo " generated/sivacor-partb-appendix.md" - echo " generated/sivacor-partb.md" echo " generated/REPLICATION-PartB-SIVACOR.md" } @@ -108,12 +106,9 @@ steps_output="$output_dir/sivacor-partb-replication-steps.md" findings_output="$output_dir/sivacor-partb-findings.md" environment_output="$output_dir/sivacor-partb-computing-environment.md" appendix_output="$output_dir/sivacor-partb-appendix.md" -empty_partb_output="$output_dir/sivacor-partb.md" - if [[ "$dry_run" != "--dry-run" ]] then mkdir -p "$output_dir" - : > "$empty_partb_output" fi "$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-computing-environment --output "$environment_output" $dry_run @@ -121,15 +116,33 @@ fi "$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-findings --output "$findings_output" $dry_run "$PYTHON" tools/get_sivacor_info.py --jsonld "$jsonld" --key sivacor-appendix --output "$appendix_output" $dry_run +template_input="$template" +tmp_template="" +if grep -Fq "You are starting *PartB*." "$template" +then + splitline=$(grep -Fn "You are starting *PartB*." "$template" | cut -f1 -d: | head -1) + tmp_template=$(mktemp) + tail -n +"$splitline" "$template" > "$tmp_template" + template_input="$tmp_template" +fi + if [[ "$dry_run" == "--dry-run" ]] then if [[ -f "$environment_output" && -f "$steps_output" && -f "$findings_output" && -f "$appendix_output" ]] then - "$PYTHON" tools/replace_placeholders.py --infile "$template" --indir "$output_dir" --outfile /dev/stdout + preview_output=$(mktemp) + "$PYTHON" tools/replace_placeholders.py --infile "$template_input" --indir "$output_dir" --outfile "$preview_output" + cat "$preview_output" + rm "$preview_output" else echo "DRY RUN: Would combine generated snippets with '$template' into '$output'." echo "DRY RUN: Would write the SIVACOR appendix snippet to '$appendix_output'." fi else - "$PYTHON" tools/replace_placeholders.py --infile "$template" --indir "$output_dir" --outfile "$output" + "$PYTHON" tools/replace_placeholders.py --infile "$template_input" --indir "$output_dir" --outfile "$output" +fi + +if [[ -n "$tmp_template" ]] +then + rm "$tmp_template" fi diff --git a/tools/get_sivacor_info.py b/tools/get_sivacor_info.py index 56da287..ff3c26e 100755 --- a/tools/get_sivacor_info.py +++ b/tools/get_sivacor_info.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.12 +#!/usr/bin/env python3 """ Parse SIVACOR JSONLD files for information. From 5ee1c8e3c617a6dcd6e4202678c616aefbab517e Mon Sep 17 00:00:00 2001 From: Danny McGuire Date: Wed, 24 Jun 2026 22:29:03 -0400 Subject: [PATCH 3/3] Copilot fix skip processing path dependency Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- automations/24_amend_report.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automations/24_amend_report.sh b/automations/24_amend_report.sh index 81bb373..0641cc3 100755 --- a/automations/24_amend_report.sh +++ b/automations/24_amend_report.sh @@ -1,6 +1,8 @@ #!/bin/bash #set -ev +[[ "${SkipProcessing:-}" == "yes" ]] && exit 0 + PYTHON=${PYTHON:-} if [[ -z "$PYTHON" ]] then @@ -20,8 +22,6 @@ then exit 1 fi -[[ "$SkipProcessing" == "yes" ]] && exit 0 - [[ -z $1 ]] && indir=generated || indir=$@ if [ ! -d "$indir" ]