Merge branch 'main' into lbconfigs

# Conflicts:
#	engine/schema/src/main/resources/META-INF/db/schema-41610to41700.sql
#	ui/public/locales/en.json

Author: Chales Queiroz

.asf.yaml

@@ -27,11 +27,16 @@ github:
     - infrastructure
     - java
     - python
-    - hosting
     - kvm
+    - libvirt
     - vsphere
+    - vmware
     - xenserver
     - xcp-ng
+    - orchestration
+    - virtualization
+    - virtual-machine
+    - kubernetes
 
   features:
     wiki: true
@@ -44,6 +49,11 @@ github:
     rebase: false
 
   collaborators:
+    - kiranchavala
+    - rajujith
+    - alexandremattioli
+    - vishesh92
+    - soreana
     - acs-robot
 
   protected_branches: ~

.github/linters/.flake8

@@ -20,6 +20,8 @@
 # E242 Tab after ','
 # E273 Tab after keyword
 # E274 Tab before keyword
+# E742 Do not define classes named 'I', 'O', or 'l'
+# E743 Do not define functions named 'I', 'O', or 'l'
 # E901 SyntaxError or IndentationError
 # E902 IOError
 # W291 Trailing whitespace
@@ -28,4 +30,4 @@
 # W391 Blank line at end of file
 
 [flake8]
-select = E223,E224,E242,E273,E274,E901,E902,W291,W292,W293,W391
+select = E223,E224,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391

.github/workflows/build.yml

@@ -63,4 +63,4 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd ..
           rm -fr nonoss
-          mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install
+          mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc)

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ concurrency:
 
 jobs:
   build:
+    if: github.repository == 'apache/cloudstack'
     runs-on: ubuntu-22.04
 
     strategy:
@@ -79,7 +80,8 @@ jobs:
                   smoke/test_metrics_api
                   smoke/test_migration
                   smoke/test_multipleips_per_nic
-                  smoke/test_nested_virtualization",
+                  smoke/test_nested_virtualization
+                  smoke/test_set_sourcenat",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -123,6 +125,7 @@ jobs:
                   smoke/test_usage
                   smoke/test_usage_events
                   smoke/test_vm_deployment_planner
+                  smoke/test_vm_schedule
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm
@@ -190,6 +193,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK
         uses: actions/setup-java@v3
@@ -214,6 +219,11 @@ jobs:
         run: |
           python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
 
+      - name: Install jacoco dependencies
+        run: |
+          wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip
+          unzip jacoco-0.8.10.zip -d jacoco
+
       - name: Env details
         run: |
           uname -a
@@ -237,7 +247,7 @@ jobs:
 
       - name: Build with Maven
         run: |
-          mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true
+          mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc)
 
       - name: Setup Simulator Prerequisites
         run: |
@@ -246,9 +256,16 @@ jobs:
           mvn -q -Pdeveloper -pl developer -Ddeploydb
           mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
 
+      - name: Generate jacoco-coverage.sh
+        run: |
+          echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh
+          find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh
+          find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh
+          echo "--xml jacoco-coverage.xml" >> jacoco-report.sh
+
       - name: Start CloudStack Management Server with Simulator
         run: |
-          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom"
+          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver"
           echo -e "\nStarting simulator"
           set +e
           mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true &
@@ -272,10 +289,20 @@ jobs:
           done
 
           echo -e "Stopping Simulator, integration tests run completed\n"
+          java -jar jacoco/lib/jacococli.jar dump --address localhost --port 36320 --destfile jacoco-it.exec
+          bash jacoco-report.sh
           mvn -Dsimulator -pl client jetty:stop 2>&1
           find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \;
 
       - name: Integration Tests Result
         run: |
           echo -e "Simulator CI Test Results: (only failures listed)\n"
           python3 ./tools/marvin/xunit-reader.py integration-test-results/
+
+      - uses: codecov/codecov-action@v3
+        with:
+          files: jacoco-coverage.xml
+          fail_ci_if_error: true
+          flags: simulator-marvin-tests
+          verbose: true
+          name: codecov

.github/workflows/codecov.yml

@@ -33,6 +33,8 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK11
         uses: actions/setup-java@v3
@@ -45,11 +47,12 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
           cd nonoss && bash -x install-non-oss.sh && cd ..
-          mvn -P quality -Dsimulator -Dnoredist clean install
+          mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc)
 
       - uses: codecov/codecov-action@v3
         with:
           files: ./client/target/site/jacoco-aggregate/jacoco.xml
           fail_ci_if_error: true
+          flags: unit-tests
           verbose: true
           name: codecov

.github/workflows/linter.yml

@@ -27,25 +27,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    permissions:
-      contents: read  # for actions/checkout to fetch code
-      statuses: write  # for github/super-linter to mark status of each linter run
-    name: Super-Linter Check
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v3
-        with:
-          # Full git history is needed to get a proper list of changed files within `super-linter`
-          fetch-depth: 0
-      - name: SuperLinter
-        uses: github/super-linter/slim@v4.9.6
-        env:
-          DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_PYTHON_FLAKE8: true
   pre-commit:
     name: Run pre-commit
     runs-on: ubuntu-22.04

.github/workflows/main-sonar-check.yml

@@ -28,6 +28,7 @@ permissions:
 
 jobs:
   build:
+    if: github.repository == 'apache/cloudstack'
     name: Main Sonar JaCoCo Build
     runs-on: ubuntu-22.04
     steps:
@@ -64,4 +65,4 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
           cd nonoss && bash -x install-non-oss.sh && cd ..
-          mvn -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack
+          mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack

.github/workflows/sonar-check.yml

@@ -17,19 +17,16 @@
 
 name: Sonar Quality Check
 
-on: [pull_request_target]
+on: [pull_request]
 
 permissions:
-  contents: read
+  contents: read # to fetch code (actions/checkout)
+  pull-requests: write # for sonar to comment on pull-request
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-permissions:
-  contents: read # to fetch code (actions/checkout)
-  pull-requests: write # for sonar to comment on pull-request
-
 jobs:
   build:
     if: github.repository == 'apache/cloudstack'
@@ -73,4 +70,4 @@ jobs:
         run: |
           git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
           cd nonoss && bash -x install-non-oss.sh && cd ..
-          mvn -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true
+          mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true

.github/workflows/ui.yml

@@ -47,8 +47,16 @@ jobs:
         run: |
           cd ui
           sudo npm install -g @vue/cli npm-check-updates
-          rm -f package-lock.json
           npm install
           npm run build
           npm run lint
           npm run test:unit
+
+      - uses: codecov/codecov-action@v3
+        with:
+          working-directory: ui
+          files: ./coverage/lcov.info
+          fail_ci_if_error: true
+          flags: uitests
+          verbose: true
+          name: codecov

.pre-commit-config.yaml

@@ -34,10 +34,39 @@ repos:
       #- id: check-merge-conflict
       - id: check-vcs-permalinks
       #- id: check-yaml
-      #- id: detect-private-key
+      - id: detect-private-key
+        exclude: >
+          (?x)
+          ^scripts/vm/systemvm/id_rsa\.cloud$|
+          ^server/src/test/java/com/cloud/keystore/KeystoreTest\.java$|
+          ^server/src/test/resources/certs/dsa_self_signed\.key$|
+          ^server/src/test/resources/certs/non_root\.key$|
+          ^server/src/test/resources/certs/root_chain\.key$|
+          ^server/src/test/resources/certs/rsa_ca_signed\.key$|
+          ^server/src/test/resources/certs/rsa_self_signed_with_pwd\.key$|
+          ^server/src/test/resources/certs/rsa_self_signed\.key$|
+          ^services/console-proxy/rdpconsole/src/test/doc/rdp-key\.pem$|
+          ^systemvm/agent/certs/localhost\.key$|
+          ^systemvm/agent/certs/realhostip\.key$
       - id: end-of-file-fixer
-        files: \.(java|md|py|txt|yaml|yml)$
+        exclude: \.vhd$
       #- id: fix-byte-order-marker
       - id: mixed-line-ending
         files: \.(java|md|py|txt|yaml|yml)$
       # - id: trailing-whitespace
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    - id: flake8
+      args: [--config, .github/linters/.flake8]
+      exclude: >
+        (?x)
+        ^agent/bindir/cloud-setup-agent\.in$|
+        ^client/bindir/cloud-update-xenserver-licenses\.in$|
+        ^cloud-cli/bindir/cloud-tool$|
+        ^python/bindir/cloud-grab-dependent-library-versions$|
+        ^python/bindir/cloud-setup-baremetal$|
+        ^scripts/vm/hypervisor/xenserver/storagePlugin$|
+        ^scripts/vm/hypervisor/xenserver/vmopspremium$|
+        ^setup/bindir/cloud-setup-encryption\.in$|
+        ^venv/.*$