diff --git a/.github/workflows/int-test-azure-workflow.yml b/.github/workflows/int-test-azure-workflow.yml index 743bb5df6..1e78e2d39 100644 --- a/.github/workflows/int-test-azure-workflow.yml +++ b/.github/workflows/int-test-azure-workflow.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + - CSPL-4601-rebased paths-ignore: - 'docs/**' - '*.md' @@ -108,6 +109,17 @@ jobs: inlineScript: | az aks update -n ${{ env.TEST_CLUSTER_NAME }} -g ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --attach-acr ${{ secrets.AZURE_CONTAINER_REGISTRY }} int-tests-appframeworkazure: + strategy: + fail-fast: false + matrix: + test: + [ + masterappframeworkc3, + managerappframeworkc3, + masterappframeworkm4, + managerappframeworkm4, + appframeworksS1, + ] runs-on: ubuntu-latest needs: setup-aks-cluster env: @@ -117,7 +129,7 @@ jobs: SPLUNK_ENTERPRISE_RELEASE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_RELEASE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator - TEST_FOCUS: azure_sanity + TEST_FOCUS: "${{ matrix.test }}" # This regex matches any string not containing integration keyword TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$" TEST_CLUSTER_PLATFORM: azure @@ -134,7 +146,6 @@ jobs: AZURE_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }} AZURE_REGION: ${{ secrets.AZURE_REGION }} CLUSTER_WIDE: "true" - # AZURE_MANAGED_ID_ENABLED: "${{ matrix.auth_method_managed_id }}" AZURE_MANAGED_ID_ENABLED: "false" steps: # Need this because apps are downloaded from S3. @@ -195,34 +206,50 @@ jobs: run: | docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Azure Kubernetes set context - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} + - name: Get AKS credentials + run: | + az aks get-credentials --resource-group ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --name ${{ env.TEST_CLUSTER_NAME }} --admin --overwrite-existing + - name: Setup long-lived service account auth + run: | + kubectl create serviceaccount ci-test-runner -n kube-system 2>/dev/null || true + kubectl create clusterrolebinding ci-test-runner-admin \ + --clusterrole=cluster-admin \ + --serviceaccount=kube-system:ci-test-runner 2>/dev/null || true + + cat <<'EOF' | kubectl apply -f - + apiVersion: v1 + kind: Secret + metadata: + name: ci-test-runner-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: ci-test-runner + type: kubernetes.io/service-account-token + EOF + + echo "Waiting for service account token..." + TOKEN="" + for i in $(seq 1 30); do + TOKEN=$(kubectl get secret ci-test-runner-token -n kube-system -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) + if [ -n "${TOKEN}" ]; then break; fi + sleep 2 + done + + if [ -n "${TOKEN}" ]; then + kubectl config set-credentials ci-test-runner --token="${TOKEN}" + kubectl config set-context --current --user=ci-test-runner + echo "Switched kubeconfig to long-lived service account token" + else + echo "WARNING: Could not create SA token, keeping admin cert auth" + fi + kubectl cluster-info - name: install metric server - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} - inlineScript: | - curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml - kubectl replace --force -f components.yaml || kubectl apply -f components.yaml + run: | + curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + kubectl replace --force -f components.yaml || kubectl apply -f components.yaml - name: install k8s dashboard - uses: Azure/aks-set-context@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} - cluster-name: ${{ env.TEST_CLUSTER_NAME }} - inlineScript: | - kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml - - name: Setup Kustomize run: | - sudo snap install kustomize - mkdir -p ./bin - cp /snap/bin/kustomize ./bin/kustomize + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml - name: Run Integration test timeout-minutes: 240 env: @@ -238,20 +265,20 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v6 with: - name: "splunk-pods-logs--artifacts-appframeworkazure" + name: "splunk-pods-logs--artifacts-${{ matrix.test }}" path: "/tmp/pod_logs/**" - name: Upload Integration Test Results if: always() uses: actions/upload-artifact@v6 with: - name: "test-report-integration-azure" + name: "test-report-integration-azure-${{ matrix.test }}" path: report-junit*.xml - name: Publish Integration Test Report uses: mikepenz/action-junit-report@v6 if: always() with: report_paths: 'report-junit*.xml' - check_name: 'Integration Test Results (Azure)' + check_name: 'Integration Test Results (Azure) - ${{ matrix.test }}' detailed_summary: true - name: Cleanup Test Case artifacts if: ${{ always() }} diff --git a/.github/workflows/int-test-gcp-workflow.yml b/.github/workflows/int-test-gcp-workflow.yml index c72c05579..752645df7 100644 --- a/.github/workflows/int-test-gcp-workflow.yml +++ b/.github/workflows/int-test-gcp-workflow.yml @@ -9,6 +9,7 @@ on: branches: - develop - main + - CSPL-4601-rebased paths-ignore: - 'docs/**' - '*.md' @@ -70,11 +71,11 @@ jobs: strategy: matrix: test_focus: - - { order: 1, name: "c3_gcp_sanity" } - - { order: 2, name: "c3_mgr_gcp_sanity" } - - { order: 3, name: "m4_gcp_sanity" } - - { order: 4, name: "m4_mgr_gcp_sanity" } - - { order: 5, name: "s1_gcp_sanity" } + - { order: 1, name: "masterappframeworkc3" } + - { order: 2, name: "managerappframeworkc3" } + - { order: 3, name: "masterappframeworkm4" } + - { order: 4, name: "managerappframeworkm4" } + - { order: 5, name: "appframeworksS1" } runs-on: ubuntu-latest needs: build-operator-image env: @@ -84,7 +85,7 @@ jobs: ARTIFACT_REGISTRY: ${{ secrets.GCP_ARTIFACT_REGISTRY }} GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} GCP_REGION: ${{ secrets.GCP_REGION }} - AWS_S3_REGION: ${{ secrets.GCP_REGION }} + GCP_STORAGE_REGION: ${{ secrets.GCP_REGION }} GCP_ZONE: ${{ secrets.GCP_ZONE }} GCP_NETWORK: default # Adjust if using a custom network GCP_SUBNETWORK: default # Adjust if using a custom subnetwork @@ -112,7 +113,7 @@ jobs: GCP_TEST_CONTAINER: ${{ secrets.GCP_TEST_CONTAINER}} GCP_INDEXES_CONTAINER: ${{ secrets.GCP_INDEXES_CONTAINER}} ECR_REPOSITORY: ${{ secrets.GCP_ARTIFACT_REGISTRY }} - GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }} + GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.GCP_ARTIFACT_REGISTRY }} steps: - name: Set Test Cluster Name run: | @@ -136,11 +137,12 @@ jobs: uses: google-github-actions/setup-gcloud@v1 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - install_components: 'kubectl' + install_components: 'kubectl,gke-gcloud-auth-plugin' - name: Set GCP Project run: | gcloud config set project ${{ env.GCP_PROJECT_ID }} + echo "USE_GKE_GCLOUD_AUTH_PLUGIN=True" >> $GITHUB_ENV - name: Create GKE Cluster run: | @@ -148,34 +150,15 @@ jobs: export GKE_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }} make cluster-up - - name: Get Kubernetes Credentials - run: | - gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - - name: Allow Pulling from Artifact Registry run: | gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - install_components: 'kubectl' - - name: Change Splunk Enterprise Image on Main Branches if: github.ref == 'refs/heads/main' run: | echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV - - name: Authenticate to GCP - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - - - name: Set GCP Project - run: | - gcloud config set project ${{ env.GCP_PROJECT_ID }} - - name: Install Kubectl uses: azure/setup-kubectl@v3 with: @@ -224,16 +207,6 @@ jobs: docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} docker push ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }} - - name: Get Kubernetes Credentials - run: | - gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }} - - - name: Get GKE Credentials - uses: google-github-actions/get-gke-credentials@v1 - with: - cluster_name: ${{ env.CLUSTER_NAME }} - location: ${{ env.GCP_ZONE }} - - name: Install Metrics Server run: | curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml @@ -245,7 +218,7 @@ jobs: - name: Setup Kustomize run: | - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.4.3 sudo mv kustomize /usr/local/bin/ - name: Verify kubectl Configuration diff --git a/Makefile b/Makefile index aaf7724b5..997513c4f 100644 --- a/Makefile +++ b/Makefile @@ -236,10 +236,9 @@ $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} KUSTOMIZE = $(LOCALBIN)/kustomize -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) - test -s $(LOCALBIN)/kustomize || curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,${KUSTOMIZE_VERSION}) $(LOCALBIN) + test -s $(LOCALBIN)/kustomize || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION} ENVTEST = $(LOCALBIN)/setup-envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework/c3/appframework_suite_test.go similarity index 78% rename from test/appframework_aws/c3/appframework_aws_suite_test.go rename to test/appframework/c3/appframework_suite_test.go index 6468b018a..46243df8c 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework/c3/appframework_suite_test.go @@ -14,6 +14,7 @@ package c3appfw import ( + "context" "os" "path/filepath" "testing" @@ -49,6 +50,7 @@ var ( downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "c3appfwPVTestApps-"+testenv.RandomDNSName(4)) + cloudBackend testenv.CloudStorageBackend ) // TestBasic is the main entry point @@ -102,26 +104,25 @@ var _ = BeforeSuite(func() { testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + ctx := context.TODO() + cloudBackend = testenv.NewCloudStorageBackend(testS3Bucket, testDataS3Bucket) + Expect(cloudBackend).NotTo(BeNil(), "failed to initialize cloud storage backend") - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + // Create a list of apps to upload + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + // Download V1 Apps + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + // Create a list of apps to upload after poll period + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + // Download V2 Apps + err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { @@ -129,10 +130,6 @@ var _ = AfterSuite(func() { Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) } - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - // Delete locally downloaded app files err := os.RemoveAll(downloadDirV1) Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework/c3/appframework_test.go similarity index 94% rename from test/appframework_aws/c3/appframework_aws_test.go rename to test/appframework/c3/appframework_test.go index 857ba1742..d61a7aada 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework/c3/appframework_test.go @@ -80,9 +80,8 @@ var _ = Describe("c3appfw test", func() { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -131,7 +130,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -163,14 +162,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -245,7 +244,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -255,19 +254,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -351,7 +350,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -384,14 +383,14 @@ var _ = Describe("c3appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -454,7 +453,7 @@ var _ = Describe("c3appfw test", func() { //############## DOWNGRADE APPS ############### // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -464,19 +463,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -566,14 +565,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -847,14 +846,14 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -906,17 +905,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -994,13 +993,13 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1009,7 +1008,7 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1017,7 +1016,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1034,7 +1033,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1046,7 +1045,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1097,7 +1096,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1108,19 +1107,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1204,14 +1203,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1219,14 +1218,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1242,7 +1241,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1253,7 +1252,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1320,7 +1319,7 @@ var _ = Describe("c3appfw test", func() { //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1331,19 +1330,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1413,7 +1412,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1422,13 +1421,13 @@ var _ = Describe("c3appfw test", func() { // Upload app to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload app to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1519,7 +1518,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1550,13 +1549,13 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1612,7 +1611,7 @@ var _ = Describe("c3appfw test", func() { // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1620,17 +1619,17 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to S3", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1761,14 +1760,14 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1819,17 +1818,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1947,14 +1946,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1962,14 +1961,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1985,7 +1984,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1996,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -2046,7 +2045,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2057,19 +2056,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2155,7 +2154,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2187,7 +2186,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2195,14 +2194,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2227,13 +2226,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2284,7 +2283,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2316,7 +2315,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2324,14 +2323,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2356,13 +2355,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2419,21 +2418,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2513,21 +2512,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2611,14 +2610,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2668,7 +2667,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + err = cloudBackend.DisableApps(ctx, downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered @@ -2685,7 +2684,7 @@ var _ = Describe("c3appfw test", func() { // Delete the file from S3 s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + err = cloudBackend.DeleteFile(ctx, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2726,20 +2725,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2766,13 +2765,13 @@ var _ = Describe("c3appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2843,20 +2842,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3PVTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2932,21 +2931,21 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3025,14 +3024,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3156,14 +3155,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3172,13 +3171,13 @@ var _ = Describe("c3appfw test", func() { // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app to S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload Technology add-on apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileListIdxc, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileListIdxc, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3271,12 +3270,12 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to S3 for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + // uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to S3 test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework/c3/manager_appframework_test.go similarity index 94% rename from test/appframework_aws/c3/manager_appframework_test.go rename to test/appframework/c3/manager_appframework_test.go index a49fd3eb4..acfb34e0c 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework/c3/manager_appframework_test.go @@ -78,9 +78,8 @@ var _ = Describe("c3appfw test", func() { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) } - // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) } if filePresentOnOperator { @@ -129,7 +128,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -161,14 +160,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -243,7 +242,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -253,19 +252,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -492,7 +491,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -525,14 +524,14 @@ var _ = Describe("c3appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -595,7 +594,7 @@ var _ = Describe("c3appfw test", func() { //############## DOWNGRADE APPS ############### // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -605,19 +604,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -707,14 +706,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -996,14 +995,14 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1055,17 +1054,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1143,13 +1142,13 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1158,7 +1157,7 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1166,7 +1165,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1183,7 +1182,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -1195,7 +1194,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1246,7 +1245,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1257,19 +1256,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1353,14 +1352,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1368,14 +1367,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1391,7 +1390,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1402,7 +1401,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -1469,7 +1468,7 @@ var _ = Describe("c3appfw test", func() { //############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1480,19 +1479,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1562,7 +1561,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1571,13 +1570,13 @@ var _ = Describe("c3appfw test", func() { // Upload app to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload app to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1668,7 +1667,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1699,13 +1698,13 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1761,7 +1760,7 @@ var _ = Describe("c3appfw test", func() { // ############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for C3 @@ -1769,17 +1768,17 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to S3", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1910,14 +1909,14 @@ var _ = Describe("c3appfw test", func() { s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1968,17 +1967,17 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2096,14 +2095,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2111,14 +2110,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2134,7 +2133,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -2145,7 +2144,7 @@ var _ = Describe("c3appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameShcCluster, @@ -2195,7 +2194,7 @@ var _ = Describe("c3appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -2206,19 +2205,19 @@ var _ = Describe("c3appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2304,7 +2303,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2336,7 +2335,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2344,14 +2343,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2376,13 +2375,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2433,7 +2432,7 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2465,7 +2464,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2473,14 +2472,14 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2505,13 +2504,13 @@ var _ = Describe("c3appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2568,21 +2567,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2662,21 +2661,21 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2760,14 +2759,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2817,7 +2816,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + err = cloudBackend.DisableApps(ctx, downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered @@ -2834,7 +2833,7 @@ var _ = Describe("c3appfw test", func() { // Delete the file from S3 s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + err = cloudBackend.DeleteFile(ctx, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2875,20 +2874,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2915,13 +2914,13 @@ var _ = Describe("c3appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2992,20 +2991,20 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3PVTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3081,21 +3080,21 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3174,14 +3173,14 @@ var _ = Describe("c3appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3305,14 +3304,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3321,13 +3320,13 @@ var _ = Describe("c3appfw test", func() { // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app to S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload Technology add-on apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileListIdxc, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileListIdxc, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -3420,12 +3419,12 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to S3 for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + // uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to S3 test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework/m4/appframework_suite_test.go similarity index 69% rename from test/appframework_aws/m4/appframework_aws_suite_test.go rename to test/appframework/m4/appframework_suite_test.go index aa21c7084..e1926392a 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework/m4/appframework_suite_test.go @@ -14,6 +14,7 @@ package m4appfw import ( + "context" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) @@ -48,44 +50,60 @@ var ( downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "m4appfwPVTestApps-"+testenv.RandomDNSName(4)) + cloudBackend testenv.CloudStorageBackend ) // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) + err := loadEnvFile() + Expect(err).ToNot(HaveOccurred(), "Error loading .env file") + sc, _ := GinkgoConfiguration() sc.Timeout = 240 * time.Minute RunSpecs(t, "Running "+testSuiteName, sc) } +func loadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + var _ = BeforeSuite(func() { var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + ctx := context.TODO() + cloudBackend = testenv.NewCloudStorageBackend(testS3Bucket, testDataS3Bucket) + Expect(cloudBackend).NotTo(BeNil(), "failed to initialize cloud storage backend") - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { @@ -93,10 +111,6 @@ var _ = AfterSuite(func() { Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) } - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - // Delete locally downloaded app files err := os.RemoveAll(downloadDirV1) Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework/m4/appframework_test.go similarity index 94% rename from test/appframework_aws/m4/appframework_aws_test.go rename to test/appframework/m4/appframework_test.go index fca638acb..a2c32fa08 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework/m4/appframework_test.go @@ -80,7 +80,7 @@ var _ = Describe("m4appfw test", func() { } // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -131,7 +131,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -161,13 +161,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -241,7 +241,7 @@ var _ = Describe("m4appfw test", func() { //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -251,19 +251,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -350,7 +350,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -380,13 +380,13 @@ var _ = Describe("m4appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -446,26 +446,26 @@ var _ = Describe("m4appfw test", func() { // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -557,13 +557,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -823,13 +823,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -877,20 +877,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -964,7 +964,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -994,13 +994,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1060,19 +1060,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1218,13 +1218,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1271,20 +1271,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1405,14 +1405,14 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1420,14 +1420,14 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1444,7 +1444,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1455,7 +1455,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1510,7 +1510,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1521,20 +1521,20 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1619,7 +1619,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1651,7 +1651,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1659,14 +1659,14 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1692,13 +1692,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1756,7 +1756,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1788,7 +1788,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1796,14 +1796,14 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1830,13 +1830,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1892,19 +1892,19 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1984,19 +1984,19 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2080,13 +2080,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2138,7 +2138,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + err = cloudBackend.DisableApps(ctx, downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered @@ -2158,7 +2158,7 @@ var _ = Describe("m4appfw test", func() { // Delete the file from S3 s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + err = cloudBackend.DeleteFile(ctx, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2198,13 +2198,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2303,20 +2303,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2383,18 +2383,18 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2420,13 +2420,13 @@ var _ = Describe("m4appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2499,20 +2499,20 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3PVTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2584,19 +2584,19 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2674,13 +2674,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework/m4/manager_appframework_test.go similarity index 94% rename from test/appframework_aws/m4/manager_appframework_test.go rename to test/appframework/m4/manager_appframework_test.go index 03e41df85..168184971 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework/m4/manager_appframework_test.go @@ -79,7 +79,7 @@ var _ = Describe("m4appfw test", func() { } // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -130,7 +130,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -160,13 +160,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -240,7 +240,7 @@ var _ = Describe("m4appfw test", func() { //############# UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -250,19 +250,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -349,7 +349,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -379,13 +379,13 @@ var _ = Describe("m4appfw test", func() { // Upload V2 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -445,26 +445,26 @@ var _ = Describe("m4appfw test", func() { // Delete V2 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V1 apps to S3 for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -556,13 +556,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -822,13 +822,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -876,20 +876,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -963,7 +963,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -993,13 +993,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1059,19 +1059,19 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Monitoring Console testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1216,13 +1216,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1269,20 +1269,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1403,14 +1403,14 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListLocal list of apps to S3 (to be used for local install) for Shc testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1418,14 +1418,14 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1442,7 +1442,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcCluster)} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: appSourceVolumeNameIdxcCluster, @@ -1453,7 +1453,7 @@ var _ = Describe("m4appfw test", func() { // Create App framework Spec for Search head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateAppFrameworkVolumeSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcCluster)} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ @@ -1508,7 +1508,7 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Redefine app lists as LDAP app isn't in V1 apps @@ -1519,20 +1519,20 @@ var _ = Describe("m4appfw test", func() { appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcLocal, localappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcLocal, localappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShcCluster, clusterappFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1617,7 +1617,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1649,7 +1649,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1657,14 +1657,14 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1690,13 +1690,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1754,7 +1754,7 @@ var _ = Describe("m4appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1786,7 +1786,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Manager @@ -1794,14 +1794,14 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big-size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1828,13 +1828,13 @@ var _ = Describe("m4appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload more apps to S3 for Deployer testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1890,19 +1890,19 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1982,19 +1982,19 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2078,13 +2078,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2136,7 +2136,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) // Disable the app - err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDirIdxc) + err = cloudBackend.DisableApps(ctx, downloadDirV1, appFileName, s3TestDirIdxc) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered @@ -2156,7 +2156,7 @@ var _ = Describe("m4appfw test", func() { // Delete the file from S3 s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + err = cloudBackend.DeleteFile(ctx, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName)) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -2196,13 +2196,13 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2301,20 +2301,20 @@ var _ = Describe("m4appfw test", func() { //############### UPGRADE APPS ################ // Delete V1 apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2381,18 +2381,18 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2418,13 +2418,13 @@ var _ = Describe("m4appfw test", func() { appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V2 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2497,20 +2497,20 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3PVTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2582,19 +2582,19 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster appVersion := "V1" testcaseEnvInst.Log.Info("Upload big size app to S3 for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Upload big size app to S3 for Search Head Cluster testcaseEnvInst.Log.Info("Upload big size app to S3 for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2672,13 +2672,13 @@ var _ = Describe("m4appfw test", func() { // Upload V1 apps to S3 for Indexer Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirIdxc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster %s", appVersion, testS3Bucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload V1 apps to S3 for Search Head Cluster testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework/s1/appframework_suite_test.go similarity index 74% rename from test/appframework_aws/s1/appframework_aws_suite_test.go rename to test/appframework/s1/appframework_suite_test.go index 252889490..d3389c5b3 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework/s1/appframework_suite_test.go @@ -14,6 +14,7 @@ package s1appfw import ( + "context" "os" "path/filepath" "testing" @@ -48,6 +49,7 @@ var ( downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) downloadDirPVTestApps = filepath.Join(currDir, "s1appfwPVTestApps-"+testenv.RandomDNSName(4)) + cloudBackend testenv.CloudStorageBackend ) // TestBasic is the main entry point @@ -59,30 +61,24 @@ func TestBasic(t *testing.T) { } var _ = BeforeSuite(func() { + ctx := context.TODO() var err error testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) Expect(err).ToNot(HaveOccurred()) - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) + cloudBackend = testenv.NewCloudStorageBackend(testS3Bucket, testDataS3Bucket) - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") + appListV1 = testenv.BasicApps + appFileList := testenv.GetAppFileList(appListV1) - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) + appFileList = testenv.GetAppFileList(appListV2) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") }) var _ = AfterSuite(func() { diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework/s1/appframework_test.go similarity index 95% rename from test/appframework_aws/s1/appframework_aws_test.go rename to test/appframework/s1/appframework_test.go index bf91e80ce..6ddcd0074 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework/s1/appframework_test.go @@ -69,9 +69,8 @@ var _ = Describe("s1appfw test", func() { if deployment != nil { deployment.Teardown() } - // Delete files uploaded to S3 if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) } if testcaseEnvInst != nil { Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) @@ -126,7 +125,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -157,7 +156,7 @@ var _ = Describe("s1appfw test", func() { // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -217,7 +216,7 @@ var _ = Describe("s1appfw test", func() { // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -225,11 +224,11 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -300,13 +299,13 @@ var _ = Describe("s1appfw test", func() { s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -376,7 +375,7 @@ var _ = Describe("s1appfw test", func() { // ############# DOWNGRADE APPS ################ // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // get revision number of the resource @@ -387,11 +386,11 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV1) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -474,10 +473,10 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -507,7 +506,7 @@ var _ = Describe("s1appfw test", func() { // Upload apps to S3 for Standalone s3TestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -659,13 +658,13 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Standalone s3TestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -723,7 +722,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone and Monitoring Console @@ -731,7 +730,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -773,12 +772,12 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 testcaseEnvInst.Log.Info("Upload ES app on S3") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -824,18 +823,18 @@ var _ = Describe("s1appfw test", func() { // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone appVersion = "V2" testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Es app to S3 for Standalone and Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -884,13 +883,13 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 testcaseEnvInst.Log.Info("Upload bigger amount of apps to S3 for this test") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -970,7 +969,7 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -999,7 +998,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) // Upload V1 apps to S3 - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1043,18 +1042,18 @@ var _ = Describe("s1appfw test", func() { //############### UPGRADE APPS ################ //Delete apps on S3 for new Apps - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil //Upload new Versioned Apps to S3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1148,13 +1147,13 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone testcaseEnvInst.Log.Info("Upload apps to S3 for 1st Standalone") appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileListStandalone1, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileListStandalone1, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1162,7 +1161,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Upload apps to S3 for 2nd Standalone") s3TestDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirStandalone2, appFileListStandalone2, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDirStandalone2, appFileListStandalone2, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1246,7 +1245,7 @@ var _ = Describe("s1appfw test", func() { appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1278,14 +1277,14 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1318,7 +1317,7 @@ var _ = Describe("s1appfw test", func() { appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) testcaseEnvInst.Log.Info("Upload more apps to S3 for Standalone") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1357,12 +1356,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1447,12 +1446,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1522,7 +1521,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1567,7 +1566,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) // Disable the app - err = testenv.DisableAppsToS3(downloadDirV1, appFileName, s3TestDir) + err = cloudBackend.DisableApps(ctx, downloadDirV1, appFileName, s3TestDir) Expect(err).To(Succeed(), "Unable to disable apps on S3") // Check for changes in App phase to determine if next poll has been triggered @@ -1582,7 +1581,7 @@ var _ = Describe("s1appfw test", func() { // Delete the file from S3 s3Filepath := filepath.Join(s3TestDir, appFileName[0]) - err = testenv.DeleteFileOnS3(testS3Bucket, s3Filepath) + err = cloudBackend.DeleteFile(ctx, s3Filepath) Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on S3 test directory", appFileName[0])) // Verify repo state is set to 2 (i.e app deleted from S3 bucket) @@ -1633,7 +1632,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1687,7 +1686,7 @@ var _ = Describe("s1appfw test", func() { // ############## UPGRADE APPS ################# // Delete apps on S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) + cloudBackend.DeleteFiles(ctx, uploadedApps) uploadedApps = nil // Upload V2 apps to S3 for Standalone @@ -1695,7 +1694,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1752,12 +1751,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone testcaseEnvInst.Log.Info("Upload apps to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1788,7 +1787,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to S3 for Standalone", appVersion)) appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV2) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1844,12 +1843,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = cloudBackend.DownloadFiles(ctx, s3PVTestApps, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirPVTestApps) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1911,12 +1910,12 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := cloudBackend.DownloadFiles(ctx, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone testcaseEnvInst.Log.Info("Upload big-size app to S3 for Standalone") - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -1987,7 +1986,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) + uploadedFiles, err := cloudBackend.UploadFiles(ctx, s3TestDirMC, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) @@ -2022,7 +2021,7 @@ var _ = Describe("s1appfw test", func() { // ################## SETUP FOR STANDALONE #################### // Upload V1 apps to S3 for Standalone testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + uploadedFiles, err = cloudBackend.UploadFiles(ctx, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go deleted file mode 100644 index 313c7c4fc..000000000 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurec3appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go deleted file mode 100644 index c4c6b4eff..000000000 --- a/test/appframework_az/c3/appframework_azure_test.go +++ /dev/null @@ -1,3308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApiV3.ClusterMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterMasterWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - // Wait for apps to reach Install phase before verification - for _, appSource := range allAppSourceInfo { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - err = testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on Azure") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterMasterRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) -}) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go deleted file mode 100644 index 8a8a1ec5c..000000000 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ /dev/null @@ -1,3308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - // Wait for apps to reach Install phase before verification - for _, appSource := range allAppSourceInfo { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - err = testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on Azure") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with ClusterManagerRef - ############# VERIFICATION ################# - * Verify ClusterManagerRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with ClusterManagerRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with ClusterManagerRef - testcaseEnvInst.Log.Info("Deploy Standalone with ClusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with ClusterManagerRef") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) -}) diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go deleted file mode 100644 index 6184e543b..000000000 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurem4appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go deleted file mode 100644 index ce5cba659..000000000 --- a/test/appframework_az/m4/appframework_azure_test.go +++ /dev/null @@ -1,2705 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - err = testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on Azure") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait for ClusterManager to reach Ready phase - testcaseEnvInst.Log.Info("Wait for ClusterManager and SearchHeadCluster to reach Ready phase") - err = testcaseEnvInst.WaitForClusterManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), cm.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterManager to reach Ready phase") - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Wait for app repo state to change, indicating poll interval has completed - testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager apps to update after poll interval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, shc.Name, shc.Kind, appSourceNameShc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Search Head Cluster apps to update after poll interval") - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) - -}) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go deleted file mode 100644 index 96d24efc2..000000000 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ /dev/null @@ -1,2705 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - err = testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on Azure") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait for ClusterManager to reach Ready phase - testcaseEnvInst.Log.Info("Wait for ClusterManager and SearchHeadCluster to reach Ready phase") - err = testcaseEnvInst.WaitForClusterManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), cm.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterManager to reach Ready phase") - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Wait for app repo state to change, indicating poll interval has completed - testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager apps to update after poll interval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, shc.Name, shc.Kind, appSourceNameShc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Search Head Cluster apps to update after poll interval") - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) - -}) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go deleted file mode 100644 index d3c06c3c4..000000000 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azures1appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go deleted file mode 100644 index 92736ef92..000000000 --- a/test/appframework_az/s1/appframework_azure_test.go +++ /dev/null @@ -1,2022 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azures1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframeworkazure, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - appFrameworkSpecMC.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - uploadedApps = nil - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on Azure - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to Azure - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - azTestDir = "azures1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to Azure App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Azure - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from Azure - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app on Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to S3 for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on Azure for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure - testcaseEnvInst.Log.Info("Upload bigger amount of apps to Azure for this test") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, smoke, appframeworkazures1, appframework: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Azure - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### UPGRADE APPS ################ - - //Delete apps on Azure for new Apps - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to Azure - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on Azure for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure for first Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileListStandalone1) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for second Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 2nd Standalone") - azTestDirStandalone2 := "azures1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirStandalone2, appFileListStandalone2) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone2) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ Upload Disabled App ########### - * Download disabled app from Azure - * Delete the app from Azure - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - err = testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDir) - Expect(err).To(Succeed(), "Unable to disable apps on Azure") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDir, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect Azure credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"azure_sa_name": []byte(testenv.RandomDNSName(5)), "azure_sa_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to Azure as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to Azure for Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to Azure for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from Azure - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - }) - }) -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go deleted file mode 100644 index 9aa061bad..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package c3gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "c3appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go deleted file mode 100644 index 3a189f7a0..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ /dev/null @@ -1,729 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - //"encoding/json" - "fmt" - "path/filepath" - - //"strings" - //"time" - - //enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - //var gcsTestDirShcLocal string - //var gcsTestDirIdxcLocal string - //var gcsTestDirShcCluster string - //var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Bucket for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to GCS for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to GCS for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to GCS for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS - * Create app source with local scope for C3 SVA (Cluster Master and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) -}) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go deleted file mode 100644 index 7c10a6b87..000000000 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ /dev/null @@ -1,3456 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - testenv.SpecifiedTestTimeout = 100000 - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework and Image Upgrade", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade the image and apps", func() { - - //################## SETUP #################### - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Upload V1 apps to Gcs for Monitoring Console - oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" - newImage := "splunk/splunk:latest" - - lm, err := deployment.DeployLicenseManager(ctx, deployment.GetName()) - cm, err := deployment.DeployClusterManager(ctx, deployment.GetName(), lm.GetName(), "", "") - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: cm.GetName(), - }, - }, - } - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - idxName := fmt.Sprintf("%s-idxc", deployment.GetName()) - shc, err := deployment.DeploySearchHeadCluster(ctx, shcName, cm.GetName(), lm.GetName(), "", mcName) - idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - - // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Monitoring Console goes to Ready phase - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // // Verify no SH in disconnected status is present on CM - testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE IMAGE ################ - - // Update LM Image - - testcaseEnvInst.Log.Info("Upgrading the License Manager Image", "Current Image", oldImage, "New Image", newImage) - lm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, lm) - Expect(err).To(Succeed(), "Failed upgrade License Manager image") - - // Update CM image - - testcaseEnvInst.Log.Info("Upgrading the Cluster Manager Image", "Current Image", oldImage, "New Image", newImage) - cm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed upgrade Cluster Manager image") - - // Update MC image - - testcaseEnvInst.Log.Info("Upgrading the Monitoring Console Image", "Current Image", oldImage, "New Image", newImage) - mc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, mc) - Expect(err).To(Succeed(), "Failed upgrade Monitoring Console image") - - // Update SHC image - - testcaseEnvInst.Log.Info("Upgrading the Search Head Cluster Image", "Current Image", oldImage, "New Image", newImage) - shc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed upgrade Search Head Cluster image") - - // // Update IDXC image - - testcaseEnvInst.Log.Info("Upgrading the Indexer Cluster Image", "Current Image", oldImage, "New Image", newImage) - idxc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed upgrade Indexer Cluster image") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY phase - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - // Wait for apps to reach Install phase before verification (local scope only) - // Cluster-scoped apps reach PhaseInstall only after bundle push completes; - // their phase is validated after VerifyAppFrameworkState below. - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope != enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for apps to reach Install phase on %s", appSource.CrName) - } - } - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Validate cluster-scoped apps also reached Install phase (bundle push already done by VerifyAppFrameworkState) - for _, appSource := range allAppSourceInfo { - if appSource.CrAppScope == enterpriseApi.ScopeCluster { - err = testcaseEnvInst.WaitForAllAppsPhase(ctx, deployment, appSource.CrName, appSource.CrKind, appSource.CrAppSourceName, appSource.CrAppList, enterpriseApi.PhaseInstall, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for cluster-scoped apps to reach Install phase on %s", appSource.CrName) - } - } - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from Gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on GCS") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Gcs - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) - gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Gcs - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Ensure that the Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Gcs - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Gcs - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Create directory for file upload to Gcs - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) - - // Deploy C3 SVA - // Deploy the Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shSpec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // //############### UPGRADE APPS ################ - // // Download ES App from Gcs - // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") - // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - // Expect(err).To(Succeed(), "Unable to download ES app") - - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) - // uploadedApps = append(uploadedApps, uploadedFiles...) - - // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Indexers go to Ready phase - // testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - // testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - // testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // // Get Pod age to check for pod resets later - // splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // //############ FINAL VERIFICATIONS ############ - - // shcAppSourceInfo.CrAppVersion = appVersion - // shcAppSourceInfo.CrAppList = esApp - // shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - // allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - // testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go deleted file mode 100644 index 8f4a28249..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package m4gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "m4appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go deleted file mode 100644 index 96759b180..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ /dev/null @@ -1,2725 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var gcsTestDirShc string - var gcsTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Master and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Master and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Master - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on GCS") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait for ClusterManager to reach Ready phase - testcaseEnvInst.Log.Info("Wait for ClusterManager and SearchHeadCluster to reach Ready phase") - err = testcaseEnvInst.WaitForClusterManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), cm.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterManager to reach Ready phase") - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Wait for app repo state to change, indicating poll interval has completed - testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager apps to update after poll interval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, shc.Name, shc.Kind, appSourceNameShc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Search Head Cluster apps to update after poll interval") - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Master - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) -}) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go deleted file mode 100644 index bc6837431..000000000 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ /dev/null @@ -1,2723 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var s3TestDirShc string - var s3TestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Monitoring Console to reach Ready phase - err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for MonitoringConsole to reach Ready phase") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from s3 - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, s3TestDirIdxc) - Expect(err).To(Succeed(), "Unable to disable apps on GCS") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for App state to update after config file change - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, s3Filepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Manager and Search Head Cluster - * Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - - // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Manager CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") - - // Set AppsRepoPollInterval for Cluster Manager to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Manager") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Manager") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Manager") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait for ClusterManager to reach Ready phase - testcaseEnvInst.Log.Info("Wait for ClusterManager and SearchHeadCluster to reach Ready phase") - err = testcaseEnvInst.WaitForClusterManagerPhase(ctx, deployment, testcaseEnvInst.GetName(), cm.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterManager to reach Ready phase") - - // Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Wait for app repo state to change, indicating poll interval has completed - testcaseEnvInst.Log.Info("Wait for app repo state to change after AppsRepoPollInterval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager apps to update after poll interval") - err = testcaseEnvInst.WaitForAppRepoStateChange(ctx, deployment, shc.Name, shc.Kind, appSourceNameShc, appListV2, 1, 3*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Search Head Cluster apps to update after poll interval") - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify no pods reset by checking the pod age - testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - }) - }) -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go deleted file mode 100644 index af2fab4c2..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package gcps1appfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "s1appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go deleted file mode 100644 index edfd8019f..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ /dev/null @@ -1,2055 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package gcps1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var gcsTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to gcs App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to gcs for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on gcs - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to gcs - appVersion := "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList := testenv.GetAppFileList(appListV2) - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - //########### SCALING UP VERIFICATION ######### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - //########### SCALING DOWN VERIFICATION ####### - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to gcs App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // ############## UPGRADE APPS ################# - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworkgcp: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to gcs - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download ES app from gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to gcs - testcaseEnvInst.Log.Info("Upload ES app on gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Ensure Standalone goes to Ready phase - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to gcs for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Es app to gcs for Standalone and Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on gcs for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info("Upload bigger amount of apps to GCS for this test") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworkgcp: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Upload V1 apps to GCS - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - //############### UPGRADE APPS ################ - - //Delete apps on GCS for new Apps - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to GCS - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on GCS for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from GCS - testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS for first Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileListStandalone1, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCS for second Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 2nd Standalone") - gcsTestDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirStandalone2, appFileListStandalone2, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standalone2Name, standalone2) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCS for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - err = wait.PollUntilContextTimeout(ctx, testenv.PollInterval, 10*time.Second, true, func(ctx context.Context) (bool, error) { - _, getErr := testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - return getErr == nil, nil - }) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - // Verify App installation is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install an app then disable it and remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - * Disable the app - * Delete the app from GCS - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - err = testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDir) - Expect(err).To(Succeed(), "Unable to disable apps on GCS") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.WaitforAppInstallState(ctx, deployment, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from GCS - gcsFilepath := filepath.Join(gcsTestDir, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGCSBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from GCS bucket) - testcaseEnvInst.VerifyAppRepoState(ctx, deployment, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect GCS credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - Expect(err).To(Succeed(), "Unable to obtain secret object") - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"gcs_access_key": []byte(testenv.RandomDNSName(5)), "gcs_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to GCS as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from GCS - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to GCS for Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to gcs for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from GCS - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Get Pod age to check for pod resets later - splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - }) - }) - -}) diff --git a/test/deploy-eks-cluster.sh b/test/deploy-eks-cluster.sh index cf30ba81e..aac0e46fa 100755 --- a/test/deploy-eks-cluster.sh +++ b/test/deploy-eks-cluster.sh @@ -140,8 +140,6 @@ function createCluster() { return 1 fi - - # Login to ECR registry so images can be push and pull from later whe # Output echo "EKS cluster nodes:" eksctl get cluster --name=${TEST_CLUSTER_NAME} diff --git a/test/deploy-gcp-cluster.sh b/test/deploy-gcp-cluster.sh index ceb7ba515..b494d1b9e 100755 --- a/test/deploy-gcp-cluster.sh +++ b/test/deploy-gcp-cluster.sh @@ -20,6 +20,47 @@ if [[ -z "${GKE_CLUSTER_K8_VERSION}" ]]; then export GKE_CLUSTER_K8_VERSION="1.29" fi +function setupServiceAccountAuth() { + echo "Creating long-lived service account for CI test runs..." + + kubectl create serviceaccount ci-test-runner -n kube-system 2>/dev/null || true + kubectl create clusterrolebinding ci-test-runner-admin \ + --clusterrole=cluster-admin \ + --serviceaccount=kube-system:ci-test-runner 2>/dev/null || true + + cat </dev/null | base64 -d 2>/dev/null) + if [ -n "${TOKEN}" ]; then + break + fi + sleep 2 + done + + if [ -z "${TOKEN}" ]; then + echo "WARNING: Failed to obtain service account token, keeping exec-based auth" + return 0 + fi + + kubectl config set-credentials ci-test-runner --token="${TOKEN}" + kubectl config set-context --current --user=ci-test-runner + + echo "Switched kubeconfig to long-lived service account token" + kubectl cluster-info +} + function deleteCluster() { echo "Cleanup remaining PVC on the GKE Cluster ${TEST_CLUSTER_NAME}" tools/cleanup.sh @@ -49,7 +90,8 @@ function createCluster() { --subnetwork ${GCP_SUBNETWORK} \ --machine-type n2-standard-8 \ --scopes "https://www.googleapis.com/auth/cloud-platform" \ - --enable-ip-alias + --enable-ip-alias \ + --cluster-version=${GKE_CLUSTER_K8_VERSION} if [ $? -ne 0 ]; then echo "Unable to create cluster - ${TEST_CLUSTER_NAME}" return 1 @@ -67,7 +109,8 @@ function createCluster() { return 1 fi - # Output + setupServiceAccountAuth + echo "GKE cluster nodes:" kubectl get nodes } diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index a1806d657..591a65c3b 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -401,7 +401,7 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob")} } case "gcp": - volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} + volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultGCPRegion())} default: testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") @@ -427,6 +427,26 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, return appFrameworkSpec } +// GenerateAppFrameworkVolumeSpec returns a VolumeSpec appropriate for the current ClusterProvider. +// Use this instead of calling GenerateIndexVolumeSpec directly with hardcoded provider values. +func GenerateAppFrameworkVolumeSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string) enterpriseApi.VolumeSpec { + switch ClusterProvider { + case "eks": + return GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region()) + case "azure": + managedID := os.Getenv("AZURE_MANAGED_ID_ENABLED") + if managedID == "false" { + return GenerateIndexVolumeSpecAzure(volumeName, GetAzureEndpoint(ctx), testenvInstance.GetIndexSecretName(), "azure", "blob") + } + return GenerateIndexVolumeSpecAzureManagedID(volumeName, GetAzureEndpoint(ctx), "azure", "blob") + case "gcp": + return GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultGCPRegion()) + default: + testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") + return enterpriseApi.VolumeSpec{} + } +} + // WaitforPhaseChange Wait for timeout or when phase change is seen on a CR for any particular app // Deprecated: Use WaitForAppPhaseChange instead for better timeout control func WaitforPhaseChange(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, crKind string, appSourceName string, appList []string) { diff --git a/test/testenv/cloud_storage_backend.go b/test/testenv/cloud_storage_backend.go new file mode 100644 index 000000000..21063aaa7 --- /dev/null +++ b/test/testenv/cloud_storage_backend.go @@ -0,0 +1,133 @@ +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" +) + +// CloudStorageBackend abstracts cloud-specific upload/delete/download +// operations so that app framework integration tests can be written +// once and parameterized by provider. +type CloudStorageBackend interface { + UploadFiles(ctx context.Context, testDir string, appFileList []string, localDir string) ([]string, error) + DeleteFiles(ctx context.Context, uploadedFiles []string) error + DeleteFile(ctx context.Context, filePath string) error + DownloadFiles(ctx context.Context, remoteDir string, localDir string, fileList []string) error + DisableApps(ctx context.Context, downloadDir string, appFileList []string, testDir string) error + GetCloudProvider() string +} + +// NewCloudStorageBackend returns the correct CloudStorageBackend implementation +// based on the current ClusterProvider. The bucket and dataBucket parameters +// are used by S3 and GCP; Azure ignores them. +func NewCloudStorageBackend(bucket, dataBucket string) CloudStorageBackend { + switch ClusterProvider { + case "eks": + return NewS3Backend(bucket, dataBucket) + case "azure": + return NewAzureBackend() + case "gcp": + return NewGCPBackend(bucket, dataBucket) + default: + Fail(fmt.Sprintf("unsupported cluster provider: %s", ClusterProvider)) + return nil + } +} + +// S3Backend implements CloudStorageBackend for AWS S3. +type S3Backend struct { + Bucket string + DataBucket string +} + +func NewS3Backend(bucket, dataBucket string) *S3Backend { + return &S3Backend{Bucket: bucket, DataBucket: dataBucket} +} + +func (b *S3Backend) UploadFiles(_ context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToS3(b.Bucket, testDir, appFileList, localDir) +} + +func (b *S3Backend) DeleteFiles(_ context.Context, uploadedFiles []string) error { + return DeleteFilesOnS3(b.Bucket, uploadedFiles) +} + +func (b *S3Backend) DownloadFiles(_ context.Context, remoteDir string, localDir string, fileList []string) error { + return DownloadFilesFromS3(b.DataBucket, remoteDir, localDir, fileList) +} + +func (b *S3Backend) DeleteFile(_ context.Context, filePath string) error { + return DeleteFileOnS3(b.Bucket, filePath) +} + +func (b *S3Backend) DisableApps(_ context.Context, downloadDir string, appFileList []string, testDir string) error { + return DisableAppsToS3(downloadDir, appFileList, testDir) +} + +func (b *S3Backend) GetCloudProvider() string { return "eks" } + +// AzureBackend implements CloudStorageBackend for Azure Blob Storage. +type AzureBackend struct{} + +func NewAzureBackend() *AzureBackend { + return &AzureBackend{} +} + +func (b *AzureBackend) UploadFiles(ctx context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToAzure(ctx, StorageAccount, StorageAccountKey, localDir, testDir, appFileList) +} + +func (b *AzureBackend) DeleteFiles(ctx context.Context, uploadedFiles []string) error { + client := &AzureBlobClient{} + return client.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedFiles) +} + +func (b *AzureBackend) DownloadFiles(ctx context.Context, remoteDir string, localDir string, fileList []string) error { + containerName := "/test-data/" + remoteDir + return DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, localDir, containerName, fileList) +} + +func (b *AzureBackend) DeleteFile(ctx context.Context, filePath string) error { + client := &AzureBlobClient{} + return client.DeleteFileOnAzure(ctx, filePath, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount) +} + +func (b *AzureBackend) DisableApps(ctx context.Context, downloadDir string, appFileList []string, testDir string) error { + return DisableAppsOnAzure(ctx, downloadDir, appFileList, testDir) +} + +func (b *AzureBackend) GetCloudProvider() string { return "azure" } + +// GCPBackend implements CloudStorageBackend for Google Cloud Storage. +type GCPBackend struct { + Bucket string + DataBucket string +} + +func NewGCPBackend(bucket, dataBucket string) *GCPBackend { + return &GCPBackend{Bucket: bucket, DataBucket: dataBucket} +} + +func (b *GCPBackend) UploadFiles(_ context.Context, testDir string, appFileList []string, localDir string) ([]string, error) { + return UploadFilesToGCP(b.Bucket, testDir, appFileList, localDir) +} + +func (b *GCPBackend) DeleteFiles(_ context.Context, uploadedFiles []string) error { + return DeleteFilesOnGCP(b.Bucket, uploadedFiles) +} + +func (b *GCPBackend) DownloadFiles(_ context.Context, remoteDir string, localDir string, fileList []string) error { + return DownloadFilesFromGCP(b.DataBucket, remoteDir, localDir, fileList) +} + +func (b *GCPBackend) DeleteFile(_ context.Context, filePath string) error { + return DeleteFileOnGCP(b.Bucket, filePath) +} + +func (b *GCPBackend) DisableApps(_ context.Context, downloadDir string, appFileList []string, testDir string) error { + return DisableAppsToGCP(b.Bucket, downloadDir, appFileList, testDir) +} + +func (b *GCPBackend) GetCloudProvider() string { return "gcp" } diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index 395722983..b42de9f05 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -216,8 +216,11 @@ func (d *Deployment) PodExecCommand(ctx context.Context, podName string, cmd []s if err != nil { return "", "", err } - //FIXME - restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), http.DefaultClient) + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return "", "", err + } + restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), httpClient) if err != nil { return "", "", err } @@ -263,8 +266,11 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, if err != nil { return "", "", err } - //FIXME - restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), http.DefaultClient) + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return "", "", err + } + restClient, err := apiutil.RESTClientForGVK(gvk, false, restConfig, serializer.NewCodecFactory(scheme.Scheme), httpClient) if err != nil { return "", "", err } diff --git a/test/testenv/gcputils.go b/test/testenv/gcputils.go index 78c41d1aa..52b61cfb8 100644 --- a/test/testenv/gcputils.go +++ b/test/testenv/gcputils.go @@ -394,7 +394,7 @@ func UploadFilesToGCP(bucketName, gcpTestDir string, appList []string, uploadDir } // DisableAppsToGCP untars apps, modifies their config files to disable them, re-tars, and uploads the disabled versions to GCP -func DisableAppsToGCP(downloadDir string, appFileList []string, gcpTestDir string) error { +func DisableAppsToGCP(bucketName string, downloadDir string, appFileList []string, gcpTestDir string) error { // Create directories for untarred and disabled apps untarredAppsMainFolder := filepath.Join(downloadDir, "untarred_apps") disabledAppsFolder := filepath.Join(downloadDir, "disabled_apps") @@ -429,8 +429,25 @@ func DisableAppsToGCP(downloadDir string, appFileList []string, gcpTestDir strin return err } + // Find the app root directory inside the extracted folder + entries, err := os.ReadDir(untarredCurrentAppFolder) + if err != nil { + logf.Log.Error(err, "Unable to read untarred app folder", "Folder", untarredCurrentAppFolder) + return err + } + var appRootDir string + for _, entry := range entries { + if entry.IsDir() { + appRootDir = filepath.Join(untarredCurrentAppFolder, entry.Name()) + break + } + } + if appRootDir == "" { + return fmt.Errorf("no app root directory found in %s", untarredCurrentAppFolder) + } + // Disable the app by modifying its config file - appConfFile := filepath.Join(untarredCurrentAppFolder, "default", "app.conf") + appConfFile := filepath.Join(appRootDir, "default", "app.conf") err = disableAppConfig(appConfFile) if err != nil { logf.Log.Error(err, "Failed to disable app config", "File", appConfFile) @@ -447,7 +464,7 @@ func DisableAppsToGCP(downloadDir string, appFileList []string, gcpTestDir strin } // Upload disabled apps to GCP - _, err = UploadFilesToGCP(testIndexesGCPBucket, gcpTestDir, appFileList, disabledAppsFolder) + _, err = UploadFilesToGCP(bucketName, gcpTestDir, appFileList, disabledAppsFolder) if err != nil { logf.Log.Error(err, "Failed to upload disabled apps to GCP") return err @@ -474,6 +491,9 @@ func untarFile(src, dest string) error { for { header, err := tarReader.Next() + if err == io.EOF { + break + } if err != nil { return err } @@ -484,15 +504,6 @@ func untarFile(src, dest string) error { return fmt.Errorf("invalid file path: %s", targetPath) } - if err == io.EOF { - break // End of archive - } - if err != nil { - return err - } - - targetPath = filepath.Join(dest, header.Name) - switch header.Typeflag { case tar.TypeDir: // Create Directory diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..c46642b75 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -116,6 +116,7 @@ func GenerateIndexVolumeSpecAzure(volumeName string, endpoint string, secretRef SecretRef: secretRef, Provider: provider, Type: storageType, + Region: Region, } } @@ -127,6 +128,7 @@ func GenerateIndexVolumeSpecAzureManagedID(volumeName string, endpoint string, p Path: azureIndexesContainer, Provider: provider, Type: storageType, + Region: Region, } } diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 5399e56b4..b6b38fd4f 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -122,30 +122,39 @@ func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { } func (testenv *TestCaseEnv) setup() error { - testenv.Log.Info("testenv initializing.\n") + testenv.Log.Info("[DEBUG] testenv setup starting", "namespace", testenv.namespace, "clusterProvider", ClusterProvider, "clusterWide", installOperatorClusterWide) var err error + testenv.Log.Info("[DEBUG] Creating namespace...") err = testenv.createNamespace() if err != nil { + testenv.Log.Info("[DEBUG] Failed to create namespace", "error", err) return err } + testenv.Log.Info("[DEBUG] Namespace created") + testenv.Log.Info("[DEBUG] Creating service account...") err = testenv.createSA() if err != nil { + testenv.Log.Info("[DEBUG] Failed to create service account", "error", err) return err } + testenv.Log.Info("[DEBUG] Service account created") if installOperatorClusterWide != "true" { + testenv.Log.Info("[DEBUG] Creating role...") err = testenv.createRole() if err != nil { return err } + testenv.Log.Info("[DEBUG] Creating role binding...") err = testenv.createRoleBinding() if err != nil { return err } + testenv.Log.Info("[DEBUG] Creating operator...") err = testenv.createOperator() if err != nil { return err @@ -153,6 +162,7 @@ func (testenv *TestCaseEnv) setup() error { } // Create secret object for index test + testenv.Log.Info("[DEBUG] Creating index secret", "provider", ClusterProvider) switch ClusterProvider { case "eks": testenv.createIndexSecret() @@ -164,15 +174,21 @@ func (testenv *TestCaseEnv) setup() error { default: testenv.Log.Info("Failed to create secret object") } + testenv.Log.Info("[DEBUG] Index secret created") if testenv.licenseFilePath != "" { + testenv.Log.Info("[DEBUG] Creating license config map", "path", testenv.licenseFilePath) err = testenv.createLicenseConfigMap() if err != nil { + testenv.Log.Info("[DEBUG] Failed to create license config map", "error", err) return err } + testenv.Log.Info("[DEBUG] License config map created") + } else { + testenv.Log.Info("[DEBUG] No license file path set, skipping license config map") } testenv.initialized = true - testenv.Log.Info("testenv initialized.\n", "namespace", testenv.namespace) + testenv.Log.Info("[DEBUG] testenv setup complete", "namespace", testenv.namespace) return nil } diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index 83576beda..d4e41b37f 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -49,7 +49,7 @@ const ( defaultSplunkImage = "splunk/splunk:latest" // defaultTestTimeout is the max timeout in seconds before async test failed. - defaultTestTimeout = 1000000 + defaultTestTimeout = 2700 // PollInterval specifies the polling interval for slow operations (waiting for full cluster readiness) PollInterval = 5 * time.Second diff --git a/test/testenv/util.go b/test/testenv/util.go index dbd849450..941d39c10 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -812,6 +812,56 @@ func DumpGetPods(ns string) []string { return splunkPods } +// DumpGetPodsWide prints detailed pod status including node and conditions +func DumpGetPodsWide(ns string) { + output, err := exec.Command("kubectl", "get", "pods", "-n", ns, "-o", "wide").Output() + if err != nil { + logf.Log.Info("[DEBUG] Failed to get pods wide", "namespace", ns, "error", err) + return + } + logf.Log.Info("[DEBUG] Pod status (wide)", "namespace", ns) + for _, line := range strings.Split(string(output), "\n") { + if line != "" { + logf.Log.Info(line) + } + } +} + +// DumpGetPVCs prints PVC status in the namespace +func DumpGetPVCs(ns string) { + output, err := exec.Command("kubectl", "get", "pvc", "-n", ns).Output() + if err != nil { + logf.Log.Info("[DEBUG] Failed to get PVCs", "namespace", ns, "error", err) + return + } + logf.Log.Info("[DEBUG] PVC status", "namespace", ns) + for _, line := range strings.Split(string(output), "\n") { + if line != "" { + logf.Log.Info(line) + } + } +} + +// DumpGetEvents prints recent events in the namespace +func DumpGetEvents(ns string) { + output, err := exec.Command("kubectl", "get", "events", "-n", ns, "--sort-by=.lastTimestamp").Output() + if err != nil { + logf.Log.Info("[DEBUG] Failed to get events", "namespace", ns, "error", err) + return + } + lines := strings.Split(string(output), "\n") + logf.Log.Info("[DEBUG] Recent events", "namespace", ns, "total", len(lines)) + start := 0 + if len(lines) > 30 { + start = len(lines) - 30 + } + for _, line := range lines[start:] { + if line != "" { + logf.Log.Info(line) + } + } +} + // DumpDescribePods prints and returns list of pods in the namespace func DumpDescribePods(ns string) []string { output, err := exec.Command("kubectl", "describe", "pods", "-n", ns).Output() diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 4b3bc1331..b828a0c86 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -907,7 +907,14 @@ func (testenv *TestCaseEnv) VerifyAppInstalled(ctx context.Context, deployment * expectedVersion = AppInfo[appName]["V1"] } testenv.Log.Info("Verify app", "pod", podName, "app", appName, "expectedVersion", expectedVersion, "versionInstalled", versionInstalled, "updated", checkupdated) - gomega.Expect(versionInstalled).Should(gomega.Equal(expectedVersion)) + gomega.Eventually(func() string { + _, ver, err := GetPodAppStatus(ctx, deployment, podName, ns, appName, clusterWideInstall) + if err != nil { + testenv.Log.Info("Retrying app version check", "pod", podName, "app", appName, "error", err) + return "" + } + return ver + }, 5*time.Minute, PollInterval).Should(gomega.Equal(expectedVersion)) } } } diff --git a/test/trigger-tests.sh b/test/trigger-tests.sh index dce729e03..415ed3b19 100644 --- a/test/trigger-tests.sh +++ b/test/trigger-tests.sh @@ -100,12 +100,12 @@ case ${CLUSTER_PROVIDER} in fi ;; "gcp") - if [[ -z "${GCP_ENTERPRISE_LICENSE_LOCATION}" ]]; then - echo "License path not set. Changing to default" + if [[ -z "${ENTERPRISE_LICENSE_LOCATION}" ]]; then + echo "License path not set. Changing to GCP default" export ENTERPRISE_LICENSE_LOCATION="${GCP_ENTERPRISE_LICENSE_LOCATION}" fi if [[ -z "${ENTERPRISE_LICENSE_LOCATION}" ]]; then - echo "License path not set. Changing to default" + echo "License path not set. Falling back to S3 path" export ENTERPRISE_LICENSE_LOCATION="${ENTERPRISE_LICENSE_S3_PATH}" fi @@ -118,11 +118,6 @@ case ${CLUSTER_PROVIDER} in echo "Test bucket not set. Changing to default" export TEST_INDEXES_S3_BUCKET="${INDEXES_S3_BUCKET}" fi - - if [[ -z "${S3_REGION}" ]]; then - echo "S3 Region not set. Changing to default" - export S3_REGION="${AWS_S3_REGION}" - fi ;; esac