From fc9c25c3720b1cea354ebc67e5efeb4b49046ae0 Mon Sep 17 00:00:00 2001 From: paulmurdoch19 Date: Fri, 2 May 2025 17:04:28 -0700 Subject: [PATCH 1/5] change pfb_to_zip filename --- pfb_to_zip/README.md | 4 ++-- pfb_to_zip/{pfb_to_zip.py => pfb_exporter.py} | 2 +- poetry.lock | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) rename pfb_to_zip/{pfb_to_zip.py => pfb_exporter.py} (99%) diff --git a/pfb_to_zip/README.md b/pfb_to_zip/README.md index 0deef9c..65ed7d2 100644 --- a/pfb_to_zip/README.md +++ b/pfb_to_zip/README.md @@ -5,7 +5,7 @@ Extract PFB to a TSVs, with the ability to filter by column, filter by filter by R and R Studio need to be installed for the INRG transformation to work. You can download them (here)[https://posit.co/download/rstudio-desktop/] # Usage -python pfb_to_zip.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./config.py -t ncit +python pfb_exporter.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./config.py -t ncit `-i` the input PFB/AVRO file `-o` the output folder path @@ -16,4 +16,4 @@ python pfb_to_zip.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./con -python pfb_to_zip.py -i ./INRG_2024_02_20240923.avro -o ./outputs/ -c ./config.py -a INRG \ No newline at end of file +python pfb_exporter.py -i ./INRG_2024_02_20240923.avro -o ./outputs/ -c ./config.py -a INRG \ No newline at end of file diff --git a/pfb_to_zip/pfb_to_zip.py b/pfb_to_zip/pfb_exporter.py similarity index 99% rename from pfb_to_zip/pfb_to_zip.py rename to pfb_to_zip/pfb_exporter.py index 2698f6a..8124fda 100644 --- a/pfb_to_zip/pfb_to_zip.py +++ b/pfb_to_zip/pfb_exporter.py @@ -123,7 +123,7 @@ def initialize(self): if not tmp_path.exists(): tmp_path.mkdir() - tmp_path = os.path.join("./", "repo") + tmp_path = os.path.join(self.tmp_folder, "/repo") if Path(tmp_path).exists(): rmtree(tmp_path, ignore_errors=False, onerror=None) diff --git a/poetry.lock b/poetry.lock index cb0075c..cec11d1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "aiofiles" @@ -620,14 +620,14 @@ files = [ [[package]] name = "gen3" -version = "4.26.1" +version = "4.27.0" description = "Gen3 CLI and Python SDK" optional = false python-versions = "<4,>=3.9" groups = ["main"] files = [ - {file = "gen3-4.26.1-py3-none-any.whl", hash = "sha256:48ea93882a1e4a08be25d789ac73c5d78ea90476d1e86ad250e48ce35d8b0c0f"}, - {file = "gen3-4.26.1.tar.gz", hash = "sha256:4572b67b499c7660d41fdb0acac6fa10bdf0e3953528710914ce6c7341b14091"}, + {file = "gen3-4.27.0-py3-none-any.whl", hash = "sha256:27ee788d6ca8fcb76583a9e40184e9c59ae42dac226b955d75240185ce6e7b0e"}, + {file = "gen3-4.27.0.tar.gz", hash = "sha256:2c2c41cc06ad8f697a4a32aba1ed953073b538a205454c55755c319da8dc437a"}, ] [package.dependencies] @@ -820,7 +820,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_full_version <= \"3.9.0\"" +markers = "python_full_version == \"3.9.0\"" files = [ {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, @@ -1978,7 +1978,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_full_version <= \"3.9.0\"" +markers = "python_full_version == \"3.9.0\"" files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, From 441ab059dafd125a7767926e716f7b7a0bdcc8e9 Mon Sep 17 00:00:00 2001 From: paulmurdoch19 Date: Fri, 2 May 2025 17:24:28 -0700 Subject: [PATCH 2/5] repo for --analysis flag needs to be dynamic for pelican --- pfb_to_zip/pfb_exporter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pfb_to_zip/pfb_exporter.py b/pfb_to_zip/pfb_exporter.py index 8124fda..4a18a64 100644 --- a/pfb_to_zip/pfb_exporter.py +++ b/pfb_to_zip/pfb_exporter.py @@ -122,8 +122,7 @@ def initialize(self): tmp_path = Path(self.analysis_path + "/analysis") if not tmp_path.exists(): tmp_path.mkdir() - - tmp_path = os.path.join(self.tmp_folder, "/repo") + tmp_path = os.path.join(self.tmp_folder, "repo") if Path(tmp_path).exists(): rmtree(tmp_path, ignore_errors=False, onerror=None) @@ -257,8 +256,7 @@ def setup_and_run_analysis(self, consortium=None): current_dir = str(Path( __file__ ).parent.absolute()) # subprocess.call(cmd_output + " --vanilla ./repo/INRG/PCDC_To_INRG_Data_Transformation.R", shell=True) - subprocess.call ([cmd_output, "--vanilla", "./repo/INRG/PCDC_To_INRG_Data_Transformation.R", os.path.join(current_dir, "repo/"), self.analysis_path + "/tsvs/", self.analysis_path + "/analysis/"]) - + subprocess.call ([cmd_output, "--vanilla", f"{self.tmp_folder}/repo/INRG/PCDC_To_INRG_Data_Transformation.R", os.path.join(self.tmp_folder, "repo/"), self.analysis_path + "/tsvs/", self.analysis_path + "/analysis/"]) # TODO cd to consortia subfolder # TODO run the script with the input and output parameters From 28b98617c13107163497120091c1c06ae6f12879 Mon Sep 17 00:00:00 2001 From: paulmurdoch19 Date: Mon, 5 May 2025 11:40:17 -0700 Subject: [PATCH 3/5] Revert "change pfb_to_zip filename" This reverts commit fc9c25c3720b1cea354ebc67e5efeb4b49046ae0. --- pfb_to_zip/README.md | 4 ++-- pfb_to_zip/{pfb_exporter.py => pfb_to_zip.py} | 0 poetry.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) rename pfb_to_zip/{pfb_exporter.py => pfb_to_zip.py} (100%) diff --git a/pfb_to_zip/README.md b/pfb_to_zip/README.md index 65ed7d2..0deef9c 100644 --- a/pfb_to_zip/README.md +++ b/pfb_to_zip/README.md @@ -5,7 +5,7 @@ Extract PFB to a TSVs, with the ability to filter by column, filter by filter by R and R Studio need to be installed for the INRG transformation to work. You can download them (here)[https://posit.co/download/rstudio-desktop/] # Usage -python pfb_exporter.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./config.py -t ncit +python pfb_to_zip.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./config.py -t ncit `-i` the input PFB/AVRO file `-o` the output folder path @@ -16,4 +16,4 @@ python pfb_exporter.py -i ./export_2023-03-27T02_42_17.avro -o ./outputs/ -c ./c -python pfb_exporter.py -i ./INRG_2024_02_20240923.avro -o ./outputs/ -c ./config.py -a INRG \ No newline at end of file +python pfb_to_zip.py -i ./INRG_2024_02_20240923.avro -o ./outputs/ -c ./config.py -a INRG \ No newline at end of file diff --git a/pfb_to_zip/pfb_exporter.py b/pfb_to_zip/pfb_to_zip.py similarity index 100% rename from pfb_to_zip/pfb_exporter.py rename to pfb_to_zip/pfb_to_zip.py diff --git a/poetry.lock b/poetry.lock index cec11d1..cb0075c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -620,14 +620,14 @@ files = [ [[package]] name = "gen3" -version = "4.27.0" +version = "4.26.1" description = "Gen3 CLI and Python SDK" optional = false python-versions = "<4,>=3.9" groups = ["main"] files = [ - {file = "gen3-4.27.0-py3-none-any.whl", hash = "sha256:27ee788d6ca8fcb76583a9e40184e9c59ae42dac226b955d75240185ce6e7b0e"}, - {file = "gen3-4.27.0.tar.gz", hash = "sha256:2c2c41cc06ad8f697a4a32aba1ed953073b538a205454c55755c319da8dc437a"}, + {file = "gen3-4.26.1-py3-none-any.whl", hash = "sha256:48ea93882a1e4a08be25d789ac73c5d78ea90476d1e86ad250e48ce35d8b0c0f"}, + {file = "gen3-4.26.1.tar.gz", hash = "sha256:4572b67b499c7660d41fdb0acac6fa10bdf0e3953528710914ce6c7341b14091"}, ] [package.dependencies] @@ -820,7 +820,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_full_version == \"3.9.0\"" +markers = "python_full_version <= \"3.9.0\"" files = [ {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, @@ -1978,7 +1978,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_full_version == \"3.9.0\"" +markers = "python_full_version <= \"3.9.0\"" files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, From 6b77fc4211eb9bc81e74ffb242730ba81997dda2 Mon Sep 17 00:00:00 2001 From: paulmurdoch19 Date: Mon, 5 May 2025 11:42:01 -0700 Subject: [PATCH 4/5] add PFBExporter to package's __init__ file --- pfb_to_zip/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pfb_to_zip/__init__.py b/pfb_to_zip/__init__.py index e69de29..1271c1e 100644 --- a/pfb_to_zip/__init__.py +++ b/pfb_to_zip/__init__.py @@ -0,0 +1 @@ +from .pfb_to_zip import PFBExporter \ No newline at end of file From 62d079d9cf5fc53fdf1cb8d45ac94c8edb028285 Mon Sep 17 00:00:00 2001 From: paulmurdoch19 Date: Mon, 5 May 2025 14:54:16 -0700 Subject: [PATCH 5/5] update check if R is installed to support linux based systems --- pfb_to_zip/pfb_to_zip.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pfb_to_zip/pfb_to_zip.py b/pfb_to_zip/pfb_to_zip.py index 4a18a64..80ac366 100644 --- a/pfb_to_zip/pfb_to_zip.py +++ b/pfb_to_zip/pfb_to_zip.py @@ -246,13 +246,18 @@ def setup_and_run_analysis(self, consortium=None): return # TODO check R is installed + #check if R is installed try: - command = 'which RScript' - cmd_output = subprocess.check_output(command, shell=True, text=True).rstrip('\n') + cmd_output = subprocess.check_output('which Rscript', shell=True, text=True).rstrip('\n') except subprocess.CalledProcessError as grepexc: print("error code", grepexc.returncode, grepexc.output) - print("R is not installed / not found!") - return + print("Rscript is not installed or found in Path!") + try: + cmd_output = subprocess.check_output('which RScript', shell=True, text=True).rstrip('\n') + except subprocess.CalledProcessError as grepexc: + print("error code", grepexc.returncode, grepexc.output) + print("R is not installed / not found!") + return current_dir = str(Path( __file__ ).parent.absolute()) # subprocess.call(cmd_output + " --vanilla ./repo/INRG/PCDC_To_INRG_Data_Transformation.R", shell=True)