QC and runtime updates

.github/workflows/python-app.yaml

@@ -1,22 +1,28 @@
-name: Python application
+name: MONSDA CI
 
-on: [pull_request]
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+  workflow_dispatch:
+    inputs:
+      run_full_pipeline:
+        description: "Run full pipeline integration test"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  monsdatest:   
-    name: monsda-test 
-    runs-on: "ubuntu-latest"
+  tests:
+    name: Unit tests (pytest)
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -el {0}
-    # Docker Hub image that `postgres-job` executes in
-    container: node:latest
-    # service containers to run with `postgres-job`
-    steps:      
-      - uses: actions/checkout@master
+    steps:
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: conda-incubator/setup-miniconda@master
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: "latest"
           activate-environment: monsda-test
@@ -28,25 +34,70 @@ jobs:
           show-channel-urls: true
           use-only-tar-bz2: false
           auto-activate-base: false
-      - name: build and install monsda
+      - name: build and install MONSDA
         run: |
           cd ${{ github.workspace }}
-          echo "Build start"          
-          python -m pip install --upgrade pip                
-          python -m pip install build                                        
-          python -m build
-          pip install dist/*.whl
-      - name: preparing test
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+          python -m pip install pytest
+      - name: run unit tests
         run: |          
-          echo "Build finished, preparing tests"                     
-          export a=$(monsda --version 2>&1 >/dev/null |sed 's/MONSDA version //g')          
-          sed -i "s/\"VERSION\": \"FIXME\"/\"VERSION\": \"$a\"/g" tests/data/config_Test.json   
-      - name: running test
-        run: |   
-          echo "Running tests" 
+          cd ${{ github.workspace }}
+          pytest -q tests/test_Utils.py
+
+  full-pipeline:
+    name: Full pipeline integration test
+    runs-on: ubuntu-latest
+    needs: tests
+    if: >-
+      (github.event_name == 'workflow_dispatch' && inputs.run_full_pipeline) ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-pipeline'))
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          activate-environment: monsda-test
+          environment-file: environment.yml
+          python-version: 3.12.2
+          channels: conda-forge,bioconda
+          allow-softlinks: true
+          channel-priority: flexible
+          show-channel-urls: true
+          use-only-tar-bz2: false
+          auto-activate-base: false
+      - name: build and install MONSDA
+        run: |
+          cd ${{ github.workspace }}
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+      - name: prepare integration test config
+        run: |
+          cd ${{ github.workspace }}
+          VERSION=$(monsda --version 2>&1 | sed 's/MONSDA version //g')
+          sed -i "s/\"VERSION\": \"FIXME\"/\"VERSION\": \"${VERSION}\"/g" tests/data/config_Test.json
+      - name: run full pipeline test
+        run: |
           cd ${{ github.workspace }}/tests
-          ln -fs data/* .    
+          ln -fs data/* .
           mkdir -p CONDALIB
-          monsda -j 6 -c config_Test.json --directory ${PWD} --use-conda --conda-prefix CONDALIB --save && echo "MONSDA test passed" || echo "MONSDA test failed"
-          #chmod +x cicd_test.sh      
-          #./cicd_test.sh
+          monsda -j 6 -c config_Test.json --directory ${PWD} --use-conda --conda-prefix CONDALIB --save
+      - name: upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: full-pipeline-debug-artifacts
+          path: |
+            ${{ github.workspace }}/tests/LOGS
+            ${{ github.workspace }}/tests/SUBSNAKES
+            ${{ github.workspace }}/tests/FASTQ
+            ${{ github.workspace }}/tests/MAPPED
+            ${{ github.workspace }}/tests/QC
+            ${{ github.workspace }}/tests/COUNTING
+            ${{ github.workspace }}/tests/PEAKS
+            ${{ github.workspace }}/tests/TRACKS

MONSDA/Params.py

@@ -1504,14 +1504,18 @@ def comparable_as_string(config: dict, subwork: str) -> str:
             logid + "no comparables found in " + subwork + ". Compare All vs. All."
         )
         groups_by_condition = list(mu.yield_from_dict("GROUPS", config))
+        log.debug(logid + "Groups by condition: " + str(groups_by_condition))
         flattened = sorted(
             set(val for sublist in groups_by_condition for val in sublist)
         )
+        log.debug(logid + "Flattened groups: " + str(flattened))
         combined = list(set(itertools.permutations(flattened, 2)))
+        log.debug(logid + "Combined groups: " + str(combined))
         complist = []
         for key, value in combined:
             complist.append(f"{key}-vs-{value}:{key}-vs-{value}")
         compstr = ",".join(complist)
+        log.debug(logid + "Comparables string: " + compstr)
         return compstr
 
 

MONSDA/Utils.py

@@ -125,6 +125,10 @@ def setup_logger(scriptname):
         exc_tb,
     )
     print("".join(tbe.format()), file=sys.stderr)
+    # Fallback logger if initialization fails (e.g., during testing)
+    scriptname = "MONSDA"
+    log = logging.getLogger(scriptname)
+    log.setLevel(logging.INFO)
 
 
 #  NestedDefaultDict
@@ -142,7 +146,14 @@ class NestedDefaultDict(collections.defaultdict):
         NestedDefaultDict
     """
     def __init__(self, *args, **kwargs):
-        super(NestedDefaultDict, self).__init__(NestedDefaultDict, *args, **kwargs)
+        default_factory = NestedDefaultDict
+        remaining_args = args
+        if args and callable(args[0]):
+            default_factory = args[0]
+            remaining_args = args[1:]
+        super(NestedDefaultDict, self).__init__(
+            default_factory, *remaining_args, **kwargs
+        )
 
     def __repr__(self):
         return repr(dict(self))
@@ -197,11 +208,7 @@ def rmempty(check: list) -> list:
     list
         list of non-empty files
     """
-    ret = list()
-    for f in check:
-        if os.path.isfile(f):
-            ret.append(f)
-    return ret
+    return [x for x in check if os.path.isfile(x)]
 
 
 @check_run
@@ -231,7 +238,8 @@ def replacer(match):
         r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
         re.DOTALL | re.MULTILINE,
     )
-    return [re.sub(pattern, replacer, x) for x in textlist]
+    cleaned = [re.sub(pattern, replacer, x) for x in textlist]
+    return [line for line in cleaned if line.strip()]
 
 
 ##############################
@@ -673,8 +681,14 @@ def find_key_for_value(val:str, dictionary:dict) -> dict.keys:
             if dict_inst(v):
                 log.debug(logid + "item" + str(v))
                 yield from find_key_for_value(val, v)
-            elif v == val or val in v:
-                yield k
+            else:
+                contains = False
+                try:
+                    contains = val in v
+                except TypeError:
+                    contains = False
+                if v == val or contains:
+                    yield k
     else:
         return dictionary
 
@@ -1053,9 +1067,13 @@ def idfromfa(id:str) -> list:
     """
     goi, chrom, strand = [None, None, None]
     try:
-        goi, chrom = id.split(":")[::2]
-        strand = str(id.split(":")[3].split("(")[1][0])
-    except:
+        parts = id.split(":", 1)
+        goi = parts[0]
+        chrom_info = parts[1]
+        chrom = chrom_info.split(".", 1)[0]
+        sm = re.search(r"\(([+-])\)", id)
+        strand = sm.group(1) if sm else "na"
+    except Exception:
         print(
             "Fasta header is not in expected format, you will loose information on strand and chromosome"
         )
@@ -1153,12 +1171,11 @@ def multi_replace(repl:str, text:str) -> str:
     str
         string with replacements
     """
-    print("MULTI: " + str(repl) + str(text))
-    # Create a regular expression  from the dictionary keys
-    regex = re.compile("(%s)" % "|".join(map(re.escape, repl.keys())))
+    # Create a regular expression from the dictionary keys
+    regex = re.compile(r"\b(%s)\b" % "|".join(map(re.escape, repl.keys())))
 
     # For each match, look-up corresponding value in dictionary
-    return regex.sub(lambda mo: dict[mo.string[mo.start() : mo.end()]], text)
+    return regex.sub(lambda mo: repl[mo.string[mo.start() : mo.end()]], text)
 
 
 @check_run
@@ -1224,16 +1241,9 @@ def add_to_innermost_key_by_list(addto:dict, toadd:str, keylist:list) -> dict:
     logid = scriptname + ".add_to_innermost_key_by_list: "
     log.debug(logid + str(addto) + ", " + str(toadd))
 
-    tconf = {}
-    for i in range(
-        len(keylist)
-    ):  # need to add options as last element to dict of unknown depth
-        tconf[keylist[i]] = {}
-        tconf = tconf[keylist[i]]
-        if i == len(keylist) - 1:
-            tconf = toadd
-
-    addto.update(tconf)
+    if not keylist:
+        return addto
+    nested_set(addto, keylist, toadd)
     return addto