Same tests as exaflow

.github/workflows/algorithm_validation_five_nodes.yaml

@@ -0,0 +1,170 @@
+name: Algorithm Validation Tests (5 Nodes)
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  run_tests_in_five_nodes:
+    name: run_tests_in_five_nodes_${{ matrix.suffix }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suffix: times_1100mb_data_expand
+            expand_factor: 400
+            pytest_workers: 1
+          - suffix: times_1100mb_data_expand_with_n16
+            expand_factor: 400
+            pytest_workers: 16
+          - suffix: normal_data_without_n
+            expand_factor: 1
+            pytest_workers: null
+          - suffix: normal_data_with_n16
+            expand_factor: 1
+            pytest_workers: 16
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.2
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load MONETDB cached image
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache/monetdb
+          key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
+          restore-keys: |
+            ${{ runner.os }}-buildx-3-monetdb-
+
+      - name: Build MONETDB docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: monetdb/Dockerfile
+          push: false
+          load: true
+          tags: madgik/exareme2_db:testing
+          cache-from: type=local,src=/tmp/.buildx-cache/monetdb
+          cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb
+
+      - name: Load RABBITMQ cached image
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache/rabbitmq
+          key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
+          restore-keys: |
+            ${{ runner.os }}-buildx-rabbitmq-
+
+      - name: Build RABBITMQ docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: rabbitmq/Dockerfile
+          push: false
+          load: true
+          tags: madgik/exareme2_rabbitmq:testing
+          cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
+          cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq
+
+      - name: Move Docker images cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Copy .deployment.toml file
+        run: cp ./tests/algorithm_validation_tests/five_node_deployment_template.toml .deployment.toml
+
+      - name: Create config files
+        run: poetry run inv create-configs
+
+      - name: Expand test CSVs (x${{ matrix.expand_factor }})
+        if: ${{ matrix.expand_factor > 1 }}
+        run: poetry run python expand_test_csvs.py --factor ${{ matrix.expand_factor }}
+
+      - name: Deploy all api
+        run: poetry run inv deploy --no-install-dep
+
+
+      - name: Load data into DBs
+        run: poetry run inv load-data
+
+      - name: Run Worker Landscape Aggregator update
+        run: curl -X POST "http://172.17.0.1:5000/wla"
+
+      - name: Controller logs
+        run: cat /tmp/exareme2/controller.out
+
+      - name: Globalworker logs
+        run: cat /tmp/exareme2/globalworker.out
+
+      - name: Localworker logs
+        run: cat /tmp/exareme2/localworker1.out
+
+      - name: Controller logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/controller.out
+
+      - name: Globalworker logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/globalworker.out
+
+      - name: Localworker logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/localworker1.out
+
+      - name: Run Exareme2 algorithm validation tests (with resource summary)
+        env:
+          PYTEST_WORKERS: ${{ matrix.pytest_workers }}
+        run: |
+          set -o pipefail
+          eval "$(python3 scripts/ci/cgroup_snapshot.py)"
+          TIME_LOG=$(mktemp)
+          set +e
+          PYTEST_CMD=(poetry run pytest tests/algorithm_validation_tests/exareme2/ --verbosity=4 -k "input1 and not input1-")
+          if [ -n "${PYTEST_WORKERS}" ]; then
+            PYTEST_CMD+=(-n "${PYTEST_WORKERS}")
+          fi
+          /usr/bin/time -v -o "$TIME_LOG" "${PYTEST_CMD[@]}"
+          TEST_EXIT_CODE=$?
+          set -e
+          REPORT_ARGS=(--title "Exareme2 algorithm validation (5 nodes, ${{ matrix.suffix }})")
+          if [ -n "${CGROUP_BASELINE_CURRENT:-}" ]; then
+            REPORT_ARGS+=(--baseline-runner-current "$CGROUP_BASELINE_CURRENT")
+          fi
+          if [ -n "${CGROUP_BASELINE_PEAK:-}" ]; then
+            REPORT_ARGS+=(--baseline-runner-peak "$CGROUP_BASELINE_PEAK")
+          fi
+          python3 scripts/ci/report_time_stats.py "$TIME_LOG" "${REPORT_ARGS[@]}"
+          cat "$TIME_LOG"
+          exit $TEST_EXIT_CODE

.github/workflows/algorithm_validation_one_node.yaml

@@ -0,0 +1,168 @@
+name: Algorithm Validation Tests (1 Node)
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  run_tests_in_one_node:
+    name: run_tests_in_one_node_${{ matrix.suffix }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suffix: times_1100mb_data_expand
+            expand_factor: 400
+            pytest_workers: 1
+          - suffix: times_1100mb_data_expand_with_n16
+            expand_factor: 400
+            pytest_workers: 16
+          - suffix: normal_data_without_n
+            expand_factor: 1
+            pytest_workers: null
+          - suffix: normal_data_with_n16
+            expand_factor: 1
+            pytest_workers: 16
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.2
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
+
+      - name: Install dependencies (if cache missed)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-ansi --no-root
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load MONETDB cached image
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache/monetdb
+          key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
+          restore-keys: |
+            ${{ runner.os }}-buildx-3-monetdb-
+
+      - name: Build MONETDB docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: monetdb/Dockerfile
+          push: false
+          load: true
+          tags: madgik/exareme2_db:testing
+          cache-from: type=local,src=/tmp/.buildx-cache/monetdb
+          cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb
+
+      - name: Load RABBITMQ cached image
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache/rabbitmq
+          key: ${{ runner.os }}-buildx-rabbitmq-${{ hashFiles('rabbitmq/**') }}
+          restore-keys: ${{ runner.os }}-buildx-rabbitmq-
+
+      - name: Build RABBITMQ docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: rabbitmq/Dockerfile
+          push: false
+          load: true
+          tags: madgik/exareme2_rabbitmq:testing
+          cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
+          cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq
+
+      - name: Move Docker images cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Copy .deployment.toml file
+        run: cp ./tests/algorithm_validation_tests/one_node_deployment_template.toml .deployment.toml
+
+      - name: Create config files
+        run: poetry run inv create-configs
+
+      - name: Expand test CSVs (x${{ matrix.expand_factor }})
+        if: ${{ matrix.expand_factor > 1 }}
+        run: poetry run python expand_test_csvs.py --factor ${{ matrix.expand_factor }}
+
+      - name: Deploy all api
+        run: poetry run inv deploy --no-install-dep
+
+      - name: Load data into DBs
+        run: poetry run inv load-data
+
+      - name: Run Worker Landscape Aggregator update
+        run: curl -X POST "http://172.17.0.1:5000/wla"
+
+      - name: Controller logs
+        run: cat /tmp/exareme2/controller.out
+
+      - name: Globalworker logs
+        run: cat /tmp/exareme2/globalworker.out
+
+      - name: Localworker logs
+        run: cat /tmp/exareme2/localworker1.out
+
+      - name: Controller logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/controller.out
+
+      - name: Globalworker logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/globalworker.out
+
+      - name: Localworker logs (post run)
+        uses: webiny/action-post-run@3.0.0
+        with:
+          run: cat /tmp/exareme2/localworker1.out
+
+      - name: Run Exareme2 algorithm validation tests (with resource summary)
+        env:
+          PYTEST_WORKERS: ${{ matrix.pytest_workers }}
+        run: |
+          set -o pipefail
+          eval "$(python3 scripts/ci/cgroup_snapshot.py)"
+          TIME_LOG=$(mktemp)
+          set +e
+          PYTEST_CMD=(poetry run pytest tests/algorithm_validation_tests/exareme2 --verbosity=4)
+          if [ -n "${PYTEST_WORKERS}" ]; then
+            PYTEST_CMD+=(-n "${PYTEST_WORKERS}")
+          fi
+          /usr/bin/time -v -o "$TIME_LOG" "${PYTEST_CMD[@]}"
+          TEST_EXIT_CODE=$?
+          set -e
+          REPORT_ARGS=(--title "Exareme2 algorithm validation (1 node, ${{ matrix.suffix }})")
+          if [ -n "${CGROUP_BASELINE_CURRENT:-}" ]; then
+            REPORT_ARGS+=(--baseline-runner-current "$CGROUP_BASELINE_CURRENT")
+          fi
+          if [ -n "${CGROUP_BASELINE_PEAK:-}" ]; then
+            REPORT_ARGS+=(--baseline-runner-peak "$CGROUP_BASELINE_PEAK")
+          fi
+          python3 scripts/ci/report_time_stats.py "$TIME_LOG" "${REPORT_ARGS[@]}"
+          cat "$TIME_LOG"
+          exit $TEST_EXIT_CODE