diff --git a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml index ee756236f..226830c31 100644 --- a/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml +++ b/.github/workflows/arm-AL2023-build-test-push-workflow-AL2023.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-AL2023-int-test-workflow.yml b/.github/workflows/arm-AL2023-int-test-workflow.yml index 39986ebca..a65cf941f 100644 --- a/.github/workflows/arm-AL2023-int-test-workflow.yml +++ b/.github/workflows/arm-AL2023-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-al2023 diff --git a/.github/workflows/arm-RHEL-build-test-push-workflow.yml b/.github/workflows/arm-RHEL-build-test-push-workflow.yml index 07848683c..f86a5e632 100644 --- a/.github/workflows/arm-RHEL-build-test-push-workflow.yml +++ b/.github/workflows/arm-RHEL-build-test-push-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-RHEL-int-test-workflow.yml b/.github/workflows/arm-RHEL-int-test-workflow.yml index 83c7e8fb9..d2a127ca8 100644 --- a/.github/workflows/arm-RHEL-int-test-workflow.yml +++ b/.github/workflows/arm-RHEL-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-rhel diff --git a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml index a22bd10c4..5c6179c20 100644 --- a/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml +++ b/.github/workflows/arm-Ubuntu-build-test-push-workflow.yml @@ -136,7 +136,7 @@ jobs: appframeworksS1, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/arm-Ubuntu-int-test-workflow.yml b/.github/workflows/arm-Ubuntu-int-test-workflow.yml index b3f133abe..8d138f5bb 100644 --- a/.github/workflows/arm-Ubuntu-int-test-workflow.yml +++ b/.github/workflows/arm-Ubuntu-int-test-workflow.yml @@ -69,7 +69,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-arm-ubuntu diff --git a/.github/workflows/build-test-push-workflow.yml b/.github/workflows/build-test-push-workflow.yml index 01e655adb..dda66569a 100644 --- a/.github/workflows/build-test-push-workflow.yml +++ b/.github/workflows/build-test-push-workflow.yml @@ -200,7 +200,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-build-test-push-workflow.yml b/.github/workflows/distroless-build-test-push-workflow.yml index 8aa6172f7..63d9bee33 100644 --- a/.github/workflows/distroless-build-test-push-workflow.yml +++ b/.github/workflows/distroless-build-test-push-workflow.yml @@ -195,7 +195,7 @@ jobs: managerappframeworkm4, managersecret, managermc, - indingsep, + indexingestionsep, licensemanager, ] runs-on: ubuntu-latest diff --git a/.github/workflows/distroless-int-test-workflow.yml b/.github/workflows/distroless-int-test-workflow.yml index 01a024247..5f16a8472 100644 --- a/.github/workflows/distroless-int-test-workflow.yml +++ b/.github/workflows/distroless-int-test-workflow.yml @@ -73,7 +73,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image-distroless diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 001a34cee..c545b69eb 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -70,7 +70,7 @@ jobs: managercrcrud, licensemanager, managerdeletecr, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/.github/workflows/manual-int-test-workflow.yml b/.github/workflows/manual-int-test-workflow.yml index 96629316e..0e3e5d357 100644 --- a/.github/workflows/manual-int-test-workflow.yml +++ b/.github/workflows/manual-int-test-workflow.yml @@ -29,7 +29,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/namespace-scope-int-workflow.yml b/.github/workflows/namespace-scope-int-workflow.yml index fc3d3554f..640f74dc6 100644 --- a/.github/workflows/namespace-scope-int-workflow.yml +++ b/.github/workflows/namespace-scope-int-workflow.yml @@ -25,7 +25,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest env: diff --git a/.github/workflows/nightly-int-test-workflow.yml b/.github/workflows/nightly-int-test-workflow.yml index 3f886dd9a..184927f04 100644 --- a/.github/workflows/nightly-int-test-workflow.yml +++ b/.github/workflows/nightly-int-test-workflow.yml @@ -66,7 +66,7 @@ jobs: managerscaling, managercrcrud, licensemanager, - indingsep, + indexingestionsep, ] runs-on: ubuntu-latest needs: build-operator-image diff --git a/test/appframework_aws/c3/appframework_aws_suite_test.go b/test/appframework_aws/c3/appframework_aws_suite_test.go index 6468b018a..963f9ecc0 100644 --- a/test/appframework_aws/c3/appframework_aws_suite_test.go +++ b/test/appframework_aws/c3/appframework_aws_suite_test.go @@ -17,24 +17,13 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/joho/godotenv" "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -42,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -55,87 +41,15 @@ var ( func TestBasic(t *testing.T) { RegisterFailHandler(Fail) - // Find and load the .env file from the current directory upwards - err := loadEnvFile() - Expect(err).ToNot(HaveOccurred(), "Error loading .env file") - - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) -} - -//func TestMain(m *testing.M) { -// Run the tests -// os.Exit(m.Run()) -//} - -// loadEnvFile traverses up the directory tree to find a .env file -func loadEnvFile() error { - // Get the current working directory - dir, err := os.Getwd() - if err != nil { - return err - } + Expect(testenv.LoadEnvFile()).ToNot(HaveOccurred(), "Error loading .env file") - // Traverse up the directory tree - for { - // Check if .env file exists in the current directory - envFile := filepath.Join(dir, ".env") - if _, err := os.Stat(envFile); err == nil { - // .env file found, load it - return godotenv.Load(envFile) - } - - // Move up to the parent directory - parentDir := filepath.Dir(dir) - if parentDir == dir { - // Reached the root directory - return nil - } - dir = parentDir - } + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 857ba1742..3eddca4e8 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -25,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -53,44 +52,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 4000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -174,7 +141,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -193,23 +160,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -274,22 +231,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -355,7 +303,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -395,7 +343,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -414,23 +362,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -483,22 +421,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -577,7 +506,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -593,14 +522,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -650,9 +573,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -670,8 +591,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -693,12 +613,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -745,9 +660,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -763,8 +676,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -814,7 +726,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -858,7 +770,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -874,14 +786,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -923,14 +829,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -953,7 +853,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1021,7 +921,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1031,7 +931,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1044,7 +944,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1063,14 +963,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1127,14 +1021,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1164,7 +1052,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1230,7 +1118,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1240,7 +1128,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1251,7 +1139,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1270,14 +1158,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1350,14 +1232,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1387,7 +1263,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1413,7 +1289,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1432,7 +1308,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1448,14 +1324,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1560,7 +1430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1573,16 +1443,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1637,14 +1501,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1724,7 +1582,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1772,7 +1630,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1787,14 +1645,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1836,14 +1688,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1907,7 +1753,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1973,7 +1819,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1983,7 +1829,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1994,7 +1840,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2012,14 +1858,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2088,14 +1928,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2187,7 +2021,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2206,7 +2040,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2316,7 +2150,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Master @@ -2335,7 +2169,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2372,11 +2206,7 @@ var _ = Describe("c3appfw test", func() { // Wait for polling interval to pass testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify all apps are installed on indexers appList = append(testenv.BigSingleApp, testenv.ExtraApps...) @@ -2419,7 +2249,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2437,7 +2267,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2458,14 +2288,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2513,7 +2337,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2531,7 +2355,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2552,14 +2376,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2622,7 +2440,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2637,14 +2455,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2726,7 +2538,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2743,7 +2555,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2787,14 +2599,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2814,7 +2620,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2843,7 +2649,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2854,7 +2660,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2863,7 +2669,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2881,14 +2687,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2932,7 +2732,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2950,7 +2750,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2974,14 +2774,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3036,7 +2830,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3061,18 +2855,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3133,7 +2923,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, masterappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3156,14 +2946,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3182,7 +2972,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) @@ -3193,7 +2983,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) @@ -3239,14 +3029,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3271,7 +3055,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index a49fd3eb4..0fab3e695 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,44 +50,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -172,7 +138,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -191,23 +157,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -272,22 +228,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -331,7 +278,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } // Upload V1 apps to S3 for Monitoring Console @@ -496,7 +443,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -536,7 +483,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -555,23 +502,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -624,22 +561,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -718,7 +646,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -734,14 +662,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -791,9 +713,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -811,8 +731,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -838,12 +757,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -890,9 +804,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -908,8 +820,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -963,7 +874,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -1007,7 +918,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1023,14 +934,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1072,14 +977,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1102,7 +1001,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1170,7 +1069,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1180,7 +1079,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1193,7 +1092,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1212,14 +1111,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1276,14 +1169,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1313,7 +1200,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1379,7 +1266,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1389,7 +1276,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1400,7 +1287,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1419,14 +1306,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1499,14 +1380,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1536,7 +1411,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1562,7 +1437,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files @@ -1581,7 +1456,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1597,14 +1472,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1709,7 +1578,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1722,16 +1591,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1786,14 +1649,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1873,7 +1730,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1921,7 +1778,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1936,14 +1793,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1985,14 +1836,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2056,7 +1901,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2122,7 +1967,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2132,7 +1977,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2143,7 +1988,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2161,14 +2006,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2237,14 +2076,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2336,7 +2169,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2355,7 +2188,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2465,7 +2298,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big-size app to S3 for Cluster Manager @@ -2484,7 +2317,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2568,7 +2401,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2586,7 +2419,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2607,14 +2440,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2662,7 +2489,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2680,7 +2507,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2701,14 +2528,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2771,7 +2592,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2786,14 +2607,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2875,7 +2690,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2892,7 +2707,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2936,14 +2751,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2963,7 +2772,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, managerappframeworkc3, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2992,7 +2801,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -3003,7 +2812,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -3012,7 +2821,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3030,14 +2839,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3081,7 +2884,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -3099,7 +2902,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3123,14 +2926,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3185,7 +2982,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3210,18 +3007,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3282,7 +3075,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -3305,14 +3098,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Download Technology add-on app from S3 testcaseEnvInst.Log.Info("Download Technology add-on app from S3") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileListIdxc) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) Expect(err).To(Succeed(), "Unable to download ES app file from S3") // Create directory for file upload to S3 @@ -3331,7 +3124,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, s3TestDirShc, 180) @@ -3342,7 +3135,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 180) @@ -3388,14 +3181,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3420,7 +3207,7 @@ var _ = Describe("c3appfw test", func() { // // Download ES App from S3 // appVersion = "V2" // testcaseEnvInst.Log.Info("Download updated ES app from S3") - // err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + // err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") // // Upload V2 ES app to S3 for Search Head Cluster diff --git a/test/appframework_aws/m4/appframework_aws_suite_test.go b/test/appframework_aws/m4/appframework_aws_suite_test.go index aa21c7084..95c94f9d0 100644 --- a/test/appframework_aws/m4/appframework_aws_suite_test.go +++ b/test/appframework_aws/m4/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,54 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index fca638acb..ca5845c9f 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +61,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +105,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -171,7 +141,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -187,19 +157,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -270,17 +231,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -313,7 +265,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,7 +306,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -390,7 +342,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -403,19 +355,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -472,17 +415,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -512,7 +446,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,7 +501,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -582,17 +516,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -634,9 +559,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +577,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -724,9 +646,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +662,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -791,7 +710,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,7 +752,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -848,14 +767,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,14 +810,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -924,7 +831,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -968,7 +875,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1004,7 +911,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1015,19 +922,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1079,17 +977,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1182,7 +1071,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1228,7 +1117,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1242,14 +1131,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1291,14 +1174,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1365,7 +1242,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1308,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,7 +1318,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1453,7 +1330,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1473,17 +1350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1553,17 +1421,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1510,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1670,7 +1529,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1683,7 +1542,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1788,7 +1647,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Cluster Master @@ -1807,7 +1666,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1821,7 +1680,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1729,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1751,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1908,7 +1767,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1920,7 +1779,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1928,17 +1787,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1962,7 +1812,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -2000,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2012,7 +1862,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2020,17 +1870,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2054,7 +1895,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,7 +1931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2102,19 +1943,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2166,7 +1998,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,7 +2040,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2222,14 +2054,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2256,8 +2082,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2092,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2351,7 +2175,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2398,7 +2222,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2410,7 +2234,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2470,7 +2294,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2499,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2510,7 +2334,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2519,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2537,14 +2361,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2380,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2402,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2600,7 +2418,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2612,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2623,17 +2441,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2657,7 +2466,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,7 +2493,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2696,7 +2505,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2518,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 03e41df85..701d7cbb4 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -170,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -186,19 +156,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -269,25 +230,13 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify Monitoring Console is Ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) // Get Pod age to check for pod resets later @@ -312,7 +261,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,7 +302,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -389,7 +338,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -402,19 +351,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -471,17 +411,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -511,7 +442,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,7 +497,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -581,17 +512,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -633,9 +555,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +573,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -723,9 +642,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +658,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -752,9 +668,6 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(ctx, idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown, idxcName) - // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -790,7 +703,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,7 +745,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -847,14 +760,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -896,14 +803,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -923,7 +824,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,7 +868,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1014,19 +915,11 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1078,17 +971,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1108,6 +992,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) + config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) Expect(err).To(Succeed(), "Unable to update config map") @@ -1124,21 +1009,6 @@ var _ = Describe("m4appfw test", func() { config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) err = deployment.UpdateCR(ctx, config) @@ -1161,7 +1031,7 @@ var _ = Describe("m4appfw test", func() { Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - // ############ VERIFY APPS UPDATED TO V2 ############# + //############### UPGRADE APPS ################ appVersion = "V2" cmAppSourceInfo.CrAppVersion = appVersion cmAppSourceInfo.CrAppList = appListV2 @@ -1180,7 +1050,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1226,7 +1096,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1240,14 +1110,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1289,14 +1153,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1340,7 +1198,7 @@ var _ = Describe("m4appfw test", func() { // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# + // ########## Verify Manual Poll disabled after the poll is triggered ################# // Verify config map set back to off after poll trigger testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) @@ -1363,8 +1221,8 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + It("integration, m4, managerappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### @@ -1429,7 +1287,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1439,7 +1297,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1451,7 +1309,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} @@ -1471,17 +1329,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,17 +1400,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1594,281 +1434,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for SearchHeadCluster to reach Ready phase - err = testcaseEnvInst.WaitForSearchHeadClusterPhase(ctx, deployment, testcaseEnvInst.GetName(), shc.Name, enterpriseApi.PhaseReady, 60*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for SearchHeadCluster to reach Ready phase") - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to S3 for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to S3 for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from S3 during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to S3 for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Download all test apps from S3 - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to S3 for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to S3 for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to S3 for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to S3 for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to S3 for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to S3 for Deployer - testcaseEnvInst.Log.Info("Upload more apps to S3 for Deployer") - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to S3 test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for polling interval to pass - testcaseEnvInst.WaitForAppInstall(ctx, deployment, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1456,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1906,7 +1472,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1918,25 +1484,16 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager - testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) + testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1517,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1539,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to S3 for Indexer Cluster @@ -1998,7 +1555,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2010,7 +1567,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2018,17 +1575,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2052,7 +1600,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2100,19 +1648,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2164,7 +1703,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,7 +1745,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2220,14 +1759,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2254,8 +1787,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +1797,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2349,7 +1880,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +1912,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to S3 for Indexer Cluster @@ -2396,7 +1927,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2408,7 +1939,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2468,7 +1999,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, managerappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2497,7 +2028,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 for Indexer Cluster @@ -2508,7 +2039,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to S3 for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) @@ -2517,7 +2048,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2560,7 +2091,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2113,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from S3 appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to S3 for Indexer Cluster @@ -2598,7 +2129,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2610,7 +2141,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2621,17 +2152,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2655,7 +2177,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4, managerappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,7 +2204,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2694,7 +2216,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2707,14 +2229,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_aws/s1/appframework_aws_suite_test.go b/test/appframework_aws/s1/appframework_aws_suite_test.go index 252889490..ef3f14fe8 100644 --- a/test/appframework_aws/s1/appframework_aws_suite_test.go +++ b/test/appframework_aws/s1/appframework_aws_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataS3Bucket = os.Getenv("TEST_BUCKET") testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - s3AppDirV1 = testenv.AppLocationV1 - s3AppDirV2 = testenv.AppLocationV2 - s3PVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "eks" { - // Create a list of apps to upload to S3 - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to S3 after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupS3AppsSuite(testSuiteName, testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_aws/s1/appframework_aws_test.go b/test/appframework_aws/s1/appframework_aws_test.go index bf91e80ce..3b044274a 100644 --- a/test/appframework_aws/s1/appframework_aws_test.go +++ b/test/appframework_aws/s1/appframework_aws_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to S3 - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.S3CloudCleanup(testS3Bucket, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,7 +100,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -164,7 +134,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +155,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -310,7 +280,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -334,7 +304,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -354,7 +324,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -481,7 +451,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -511,7 +481,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -531,7 +501,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -566,8 +536,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -599,8 +568,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -669,7 +637,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -686,7 +654,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -706,8 +674,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -773,7 +740,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to S3 @@ -782,7 +749,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, s3TestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -805,7 +772,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -829,7 +796,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from S3 testcaseEnvInst.Log.Info("Download updated ES app from S3") - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to S3 for Standalone @@ -884,7 +851,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download bigger amount of apps from S3 for this test") - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,7 +861,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -974,7 +941,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +970,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 0) @@ -1023,7 +990,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1148,7 +1115,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from S3 testcaseEnvInst.Log.Info("Download the extra apps from S3 for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 for first Standalone @@ -1166,7 +1133,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1180,7 +1147,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, s3TestDirStandalone2, 60) @@ -1250,7 +1217,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -1278,7 +1245,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from S3 appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1289,7 +1256,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1309,7 +1276,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1357,7 +1324,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1366,7 +1333,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1383,7 +1350,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1447,7 +1414,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to S3 for Standalone @@ -1456,7 +1423,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1473,7 +1440,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1529,7 +1496,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1547,7 +1514,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1609,7 +1576,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to S3 App Source @@ -1640,7 +1607,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1659,7 +1626,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1679,7 +1646,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1752,7 +1719,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to S3 for Standalone @@ -1761,7 +1728,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1778,7 +1745,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1844,7 +1811,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3PVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to S3 @@ -1856,7 +1823,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1911,7 +1878,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to S3 for Standalone @@ -1920,7 +1887,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to S3 test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1937,7 +1904,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1991,7 +1958,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -2029,7 +1996,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, s3TestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2050,7 +2017,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go index 313c7c4fc..83875f709 100644 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ b/test/appframework_az/c3/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurec3appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c4c6b4eff..b20c56c44 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -49,43 +48,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -171,7 +138,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -190,23 +157,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -264,22 +221,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -345,7 +293,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -385,7 +333,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -404,23 +352,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -474,22 +412,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -568,7 +497,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -584,14 +513,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -631,9 +554,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -651,8 +572,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -674,12 +594,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -718,9 +633,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -736,8 +649,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -787,7 +699,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -831,7 +743,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -847,14 +759,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -897,14 +803,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -927,7 +827,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -950,7 +850,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -977,7 +877,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) @@ -988,7 +888,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1034,14 +934,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1071,7 +965,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1084,14 +978,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1108,7 +996,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1176,7 +1064,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1186,7 +1074,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1199,7 +1087,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1218,14 +1106,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1299,14 +1181,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1336,7 +1212,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1405,7 +1281,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1415,7 +1291,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1426,7 +1302,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1445,14 +1321,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1514,14 +1384,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,7 +1415,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1441,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,7 +1461,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1613,14 +1477,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1725,7 +1583,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1738,16 +1596,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1803,14 +1655,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1890,7 +1736,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,7 +1784,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1953,14 +1799,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2003,14 +1843,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2074,7 +1908,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1974,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,7 +1984,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2161,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2179,14 +2013,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2256,14 +2084,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2177,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,7 +2197,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2485,7 +2307,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,7 +2327,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2589,7 +2411,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,7 +2430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2629,14 +2451,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2500,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,7 +2519,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2724,14 +2540,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2795,7 +2605,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2810,14 +2620,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2705,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,7 +2723,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2963,14 +2767,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2990,7 +2788,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3026,7 +2824,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,7 +2833,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3053,14 +2851,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2896,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,7 +2915,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3147,14 +2939,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3209,7 +2995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3234,18 +3020,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 8a8a1ec5c..17092862c 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -48,43 +47,11 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -169,7 +136,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -188,23 +155,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -262,22 +219,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -343,7 +291,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -383,7 +331,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -402,23 +350,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -472,22 +410,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -566,7 +495,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -582,14 +511,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -629,9 +552,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -649,8 +570,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -672,12 +592,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -716,9 +631,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -734,8 +647,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -785,7 +697,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -829,7 +741,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -845,14 +757,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -895,14 +801,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -925,7 +825,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps @@ -948,7 +848,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app file from Azure") @@ -975,7 +875,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) @@ -986,7 +886,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1032,14 +932,8 @@ var _ = Describe("c3appfw test", func() { shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1069,7 +963,7 @@ var _ = Describe("c3appfw test", func() { // Download ES App from Azure appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -1082,14 +976,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - // // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1107,7 +995,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps @@ -1175,7 +1063,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1185,7 +1073,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1198,7 +1086,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1217,14 +1105,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1298,14 +1180,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1335,7 +1211,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps @@ -1404,7 +1280,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,7 +1290,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1425,7 +1301,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1444,14 +1320,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1514,14 +1384,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1551,7 +1415,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps @@ -1577,7 +1441,7 @@ var _ = Describe("c3appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1597,7 +1461,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1613,14 +1477,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1725,7 +1583,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1738,16 +1596,10 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1803,14 +1655,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1890,7 +1736,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps @@ -1938,7 +1784,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1953,14 +1799,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2003,14 +1843,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2074,7 +1908,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -2140,7 +1974,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2150,7 +1984,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2161,7 +1995,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2179,14 +2013,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2256,14 +2084,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2355,7 +2177,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2375,7 +2197,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2485,7 +2307,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2505,7 +2327,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2589,7 +2411,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2608,7 +2430,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2629,14 +2451,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2684,7 +2500,7 @@ var _ = Describe("c3appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2703,7 +2519,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2724,14 +2540,8 @@ var _ = Describe("c3appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2795,7 +2605,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2810,14 +2620,8 @@ var _ = Describe("c3appfw test", func() { cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2901,7 +2705,7 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2919,7 +2723,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2963,14 +2767,8 @@ var _ = Describe("c3appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2990,7 +2788,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -3026,7 +2824,7 @@ var _ = Describe("c3appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) @@ -3035,7 +2833,7 @@ var _ = Describe("c3appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3053,14 +2851,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3104,7 +2896,7 @@ var _ = Describe("c3appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -3123,7 +2915,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3147,14 +2939,8 @@ var _ = Describe("c3appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -3209,7 +2995,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3234,18 +3020,14 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go index 6184e543b..a416bedc9 100644 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ b/test/appframework_az/m4/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azurem4appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -53,53 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go index ce5cba659..e8ede677d 100644 --- a/test/appframework_az/m4/appframework_azure_test.go +++ b/test/appframework_az/m4/appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,31 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -171,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -187,28 +156,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -259,17 +215,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -302,7 +249,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -343,7 +290,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -379,7 +326,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -392,19 +339,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -462,17 +400,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -502,7 +431,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -557,7 +486,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -572,17 +501,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -614,9 +534,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -634,8 +552,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -696,9 +613,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -714,8 +629,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -763,7 +677,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -805,7 +719,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) @@ -820,14 +734,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -870,14 +778,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,7 +799,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -941,7 +843,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -977,7 +879,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) @@ -988,19 +890,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1052,17 +945,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1154,7 +1038,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1200,7 +1084,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -1214,14 +1098,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1264,14 +1142,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1338,7 +1210,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1404,7 +1276,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1414,7 +1286,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1426,7 +1298,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1446,17 +1318,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1527,17 +1390,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1625,7 +1479,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1645,7 +1499,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1658,7 +1512,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1762,7 +1616,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1796,7 +1650,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1845,7 +1699,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1867,7 +1721,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,7 +1738,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1896,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1904,17 +1758,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1938,7 +1783,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1960,7 +1805,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,7 +1822,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1989,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1997,17 +1842,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2031,7 +1867,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2068,7 +1904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2080,19 +1916,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2146,7 +1973,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,7 +2015,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -2202,14 +2029,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2236,8 +2057,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2067,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2332,7 +2151,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2364,7 +2183,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,7 +2199,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) @@ -2392,7 +2211,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2423,17 +2242,8 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2452,7 +2262,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2488,7 +2298,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,7 +2307,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2515,14 +2325,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2540,7 +2344,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2562,7 +2366,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,7 +2383,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2591,7 +2395,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2602,17 +2406,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2636,7 +2431,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2663,7 +2458,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2675,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2483,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go index 96d24efc2..d82223754 100644 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ b/test/appframework_az/m4/manager_appframework_azure_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,16 +50,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -69,28 +59,7 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { @@ -134,7 +103,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -170,7 +139,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -186,28 +155,15 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -258,17 +214,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -342,7 +289,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -378,7 +325,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -391,19 +338,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -461,17 +399,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -556,7 +485,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -571,17 +500,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -613,9 +533,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -633,8 +551,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -695,9 +612,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -713,8 +628,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -762,7 +676,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -804,7 +718,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) @@ -819,14 +733,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -869,14 +777,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -896,7 +798,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -940,7 +842,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -976,7 +878,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) @@ -987,19 +889,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1051,17 +944,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1153,7 +1037,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1199,7 +1083,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -1213,14 +1097,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1263,14 +1141,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1337,7 +1209,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1403,7 +1275,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1413,7 +1285,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1425,7 +1297,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} @@ -1445,17 +1317,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1526,17 +1389,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1624,7 +1478,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1644,7 +1498,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1657,7 +1511,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1762,7 +1616,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1782,7 +1636,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1796,7 +1650,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1867,7 +1721,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1884,7 +1738,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1896,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1904,17 +1758,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1960,7 +1805,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -1977,7 +1822,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -1989,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1997,17 +1842,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2068,7 +1904,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2080,19 +1916,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2146,7 +1973,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2188,7 +2015,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) @@ -2202,14 +2029,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2236,8 +2057,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2247,8 +2067,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2364,7 +2183,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -2380,7 +2199,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) @@ -2392,7 +2211,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2423,17 +2242,8 @@ var _ = Describe("m4appfw test", func() { appFileList = testenv.GetAppFileList(appListV2) testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2488,7 +2298,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") @@ -2497,7 +2307,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2515,14 +2325,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2366,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from Azure appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") @@ -2579,7 +2383,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2591,7 +2395,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2602,17 +2406,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2663,7 +2458,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) @@ -2675,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2688,14 +2483,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go index d3c06c3c4..3eff208c0 100644 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ b/test/appframework_az/s1/appframework_azure_suite_test.go @@ -14,11 +14,9 @@ package azures1appfw import ( - "context" "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -43,9 +32,6 @@ var ( AzureDataContainer = os.Getenv("TEST_CONTAINER") AzureContainer = os.Getenv("INDEXES_CONTAINER") AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -53,49 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } + testenvInstance, appListV1, appListV2 = testenv.SetupAzureAppsSuite(testSuiteName, downloadDirV1, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go index 92736ef92..d43a24954 100644 --- a/test/appframework_az/s1/appframework_azure_test.go +++ b/test/appframework_az/s1/appframework_azure_test.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -42,45 +41,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") azTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.AzureCloudCleanup(ctx, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -131,7 +99,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -164,7 +132,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +153,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -300,7 +268,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -324,7 +292,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -344,7 +312,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -474,7 +442,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -505,7 +473,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -525,7 +493,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -550,8 +518,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -575,8 +542,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -646,7 +612,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -663,7 +629,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -683,8 +649,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -751,7 +716,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from Azure") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -762,7 +727,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -785,7 +750,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -805,7 +770,7 @@ var _ = Describe("s1appfw test", func() { azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 + containerName = "/" + AzureDataContainer + "/" + testenv.AppLocationV2 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") @@ -861,7 +826,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -872,7 +837,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -952,7 +917,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) @@ -981,7 +946,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) @@ -1001,7 +966,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1115,7 +1080,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from Azure testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1134,7 +1099,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1148,7 +1113,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) @@ -1218,7 +1183,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -1246,7 +1211,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from Azure appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1258,7 +1223,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1278,7 +1243,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1326,7 +1291,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1337,7 +1302,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1354,7 +1319,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1397,7 +1362,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1408,7 +1373,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1425,7 +1390,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1482,7 +1447,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1500,7 +1465,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1564,7 +1529,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to Azure App Source @@ -1595,7 +1560,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1614,7 +1579,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) secretData := secretStruct.Data @@ -1633,7 +1598,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1707,7 +1672,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download apps") @@ -1717,7 +1682,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1734,7 +1699,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1811,7 +1776,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1866,7 +1831,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 + containerName := "/" + AzureDataContainer + "/" + testenv.AppLocationV1 err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) Expect(err).To(Succeed(), "Unable to download big app") @@ -1876,7 +1841,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1893,7 +1858,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1947,7 +1912,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) @@ -1985,7 +1950,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2006,7 +1971,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go index 9aa061bad..c73826a7a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) @@ -52,51 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go index 3a189f7a0..a9250959a 100644 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ b/test/appframework_gcp/c3/appframework_gcs_test.go @@ -17,7 +17,6 @@ import ( "context" //"encoding/json" "fmt" - "path/filepath" //"strings" //"time" @@ -26,7 +25,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -54,44 +52,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -175,7 +141,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -194,23 +160,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Verify no SH in disconnected status is present on CM testcaseEnvInst.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment) @@ -275,22 +231,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -356,7 +303,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -396,7 +343,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -415,23 +362,13 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -484,22 +421,13 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -522,7 +450,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, masterappframeworkc3gcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -566,7 +494,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -582,14 +510,8 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -631,14 +553,8 @@ var _ = Describe("c3appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -691,7 +607,7 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -716,11 +632,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go index 7c10a6b87..cb492d28e 100644 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ b/test/appframework_gcp/c3/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -51,45 +50,12 @@ var _ = Describe("c3appfw test", func() { ctx := context.TODO() BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") testenv.SpecifiedTestTimeout = 100000 }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { @@ -97,10 +63,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -112,7 +78,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -125,13 +91,13 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -159,21 +125,21 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -242,32 +208,32 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -322,7 +288,7 @@ var _ = Describe("c3appfw test", func() { switch testenv.ClusterProvider { case "eks": licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") + Expect(err).To(Succeed(), "Unable to download license file from GCS") // Create License Config Map testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) case "azure": @@ -337,10 +303,10 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) default: fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with provider set as %v", testenv.ClusterProvider)) } - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" newImage := "splunk/splunk:latest" @@ -464,10 +430,10 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console + * Upload V2 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster * Create app source for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ########### INITIAL VERIFICATIONS ########### @@ -479,7 +445,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for Monitoring Console and C3 pods to be ready ########### FINAL VERIFICATIONS ############# * Verify Apps are Downloaded in App Deployment Info @@ -492,16 +458,16 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console + // Upload V2 apps to GCS for Monitoring Console appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -527,21 +493,21 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V2 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -598,32 +564,32 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V1 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -672,7 +638,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps on GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app config and wait for pods to be ready ########## INITIAL VERIFICATIONS ############ @@ -707,23 +673,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -796,9 +762,7 @@ var _ = Describe("c3appfw test", func() { //############# SCALING UP ################### // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -816,8 +780,7 @@ var _ = Describe("c3appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-idxc" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -839,12 +802,7 @@ var _ = Describe("c3appfw test", func() { Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } + testenv.IngestDataOnIndexers(ctx, deployment, deployment.GetName(), int(scaledIndexerReplicas)) // Ensure Search Head Cluster go to Ready phase testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) @@ -891,9 +849,7 @@ var _ = Describe("c3appfw test", func() { //############### SCALING DOWN ################ // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -909,8 +865,7 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -960,12 +915,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It(" c3gcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATIONS ########## @@ -988,23 +943,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1050,20 +1005,20 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1099,13 +1054,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1117,7 +1072,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1135,39 +1090,39 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1177,7 +1132,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1190,7 +1145,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1241,8 +1196,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1250,24 +1205,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1310,13 +1265,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -1328,7 +1283,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### Downgrade APPS ################ - * Upload V1 apps on Gcs + * Upload V1 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -1346,37 +1301,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV2[len(appListV2)/2:] appListCluster := appListV2[:len(appListV2)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1386,7 +1341,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -1397,7 +1352,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -1464,8 +1419,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -1473,24 +1428,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) + // Upload appListLocal list of V1 apps to GCS (to be used for local install) appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1533,13 +1488,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { /* Test Steps ################## SETUP #################### * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test + * Add more apps than usual on GCS for this test * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############### VERIFICATIONS ############### * Verify Apps are Downloaded in App Deployment Info @@ -1557,28 +1512,28 @@ var _ = Describe("c3appfw test", func() { appFileList := testenv.GetAppFileList(appList) appVersion := "V1" - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + // Download apps from GCS + testcaseEnvInst.Log.Info("Download bigger amount of apps from GCS for this test") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Create consolidated list of app files appList = append(appListV1, appList...) appFileList = testenv.GetAppFileList(appList) - // Upload app to Gcs for Indexer Cluster + // Upload app to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload app to Gcs for Search Head Cluster + // Upload app to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1629,10 +1584,10 @@ var _ = Describe("c3appfw test", func() { It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ########## INITIAL VERIFICATION ############# @@ -1660,13 +1615,13 @@ var _ = Describe("c3appfw test", func() { */ // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -1694,19 +1649,19 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1719,7 +1674,7 @@ var _ = Describe("c3appfw test", func() { shReplicas := 3 testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App Framework") // Ensure Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1756,28 +1711,28 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs for C3 + // Upload V2 apps to GCS for C3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to GCS", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + // Upload V2 apps to GCS for Monitoring Console + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -1870,12 +1825,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs + * Upload V1 apps to GCS * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############# INITIAL VERIFICATION ########## @@ -1902,23 +1857,23 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster + // Upload V1 apps to GCS for Search Head Cluster gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1963,20 +1918,20 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete V1 apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil - // Upload V2 apps to Gcs + // Upload V2 apps to GCS appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) appFileList = testenv.GetAppFileList(appListV2) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Check for changes in App phase to determine if next poll has been triggered @@ -2053,13 +2008,13 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps ################## SETUP #################### * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster for local and cluster scope * Create app sources for Cluster Manager and Deployer with local and cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2071,7 +2026,7 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs + * Upload V2 apps on GCS * Wait for all C3 pods to be ready ############ FINAL VERIFICATIONS ############ * Verify Apps are Downloaded in App Deployment Info @@ -2089,37 +2044,37 @@ var _ = Describe("c3appfw test", func() { appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Idxc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) localappFileList := testenv.GetAppFileList(appListLocal) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + // Upload appListLocal list of apps to GCS (to be used for local install) for Shc + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) + // Upload appListCluster list of apps to GCS (to be used for cluster-wide install) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for cluster-wide install (cluster scope)", appVersion)) gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -2129,7 +2084,7 @@ var _ = Describe("c3appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) @@ -2140,7 +2095,7 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) @@ -2190,8 +2145,8 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyNoPodResetByUID(ctx, deployment, splunkPodUIDs, nil) //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) + // Delete apps on GCS + testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) uploadedApps = nil @@ -2199,24 +2154,24 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) + // Upload appListLocal list of V2 apps to GCS (to be used for local install) appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for local install (local scope)", appVersion)) localappFileList = testenv.GetAppFileList(appListLocal) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) + // Upload appListCluster list of V2 apps to GCS (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // ############ ENABLE MANUAL POLL ############ @@ -2282,27 +2237,27 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework ############## VERIFICATIONS ################ * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -2330,29 +2285,29 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2369,18 +2324,18 @@ var _ = Describe("c3appfw test", func() { // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2411,27 +2366,27 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console + * Upload V1 apps to GCS for Monitoring Console * Create app source for Monitoring Console * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ############## VERIFICATIONS ################ * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install + * Upload more apps from GCS during bigger app install * Wait for polling interval to pass * Verify all apps are installed on Cluster Manager and Deployer */ //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console + // Upload V1 apps to GCS for Monitoring Console appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) // Prepare Monitoring Console spec with its own app source @@ -2459,29 +2414,29 @@ var _ = Describe("c3appfw test", func() { // Verify Monitoring Console is ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big-size app to Gcs for Cluster Manager + // Upload big-size app to GCS for Cluster Manager appList = testenv.BigSingleApp appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Cluster Manager") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") + // Upload big-size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big-size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2498,18 +2453,18 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - // Upload more apps to Gcs for Cluster Manager + // Upload more apps to GCS for Cluster Manager appList = testenv.ExtraApps appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") + testcaseEnvInst.Log.Info("Upload more apps to GCS for Cluster Manager") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Cluster Manager") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") + // Upload more apps to GCS for Deployer + testcaseEnvInst.Log.Info("Upload more apps to GCS for Deployer") uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") + Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Deployer") uploadedApps = append(uploadedApps, uploadedFiles...) // Ensure Cluster Manager goes to Ready phase @@ -2547,7 +2502,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app install is in progress, restart the operator @@ -2562,28 +2517,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2641,7 +2596,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is in progress, restart the operator @@ -2656,28 +2611,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload V1 apps to Gcs for Indexer Cluster + // Upload V1 apps to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2735,7 +2690,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### VERIFICATIONS ############# @@ -2747,28 +2702,28 @@ var _ = Describe("c3appfw test", func() { * Verify bundle push is successful * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods * Disable the app - * Delete the app from Gcs + * Delete the app from GCS * Check for repo state in App Deployment Info */ //################## SETUP #################### appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2808,7 +2763,7 @@ var _ = Describe("c3appfw test", func() { allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) + // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) appName := appListV1[0] appFileName := testenv.GetAppFileList([]string{appName}) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) @@ -2829,12 +2784,12 @@ var _ = Describe("c3appfw test", func() { // Wait for App state to update after config file change testcaseEnvInst.WaitforAppInstallState(ctx, deployment, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - // Delete the file from Gcs + // Delete the file from GCS gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) + // Verify repo state is set to 2 (i.e app deleted from GCS bucket) testcaseEnvInst.VerifyAppRepoState(ctx, deployment, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) }) @@ -2845,7 +2800,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * While app download is completed, upload new versions of the apps @@ -2868,28 +2823,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download all apps from Gcs + // Download all apps from GCS appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2907,19 +2862,19 @@ var _ = Describe("c3appfw test", func() { // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - // Upload V2 apps to Gcs for Indexer Cluster + // Upload V2 apps to GCS for Indexer Cluster appVersion = "V2" appListV2 := []string{appListV2[0]} appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V2 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Get Pod age to check for pod resets later @@ -2960,12 +2915,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("c3, integration, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope + * Upload 15 apps of 100MB size each to GCS for Indexer Cluster and Search Head Cluster for cluster scope * Create app sources for Cluster Master and Deployer with cluster scope * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready ######### INITIAL VERIFICATIONS ############# @@ -2989,27 +2944,27 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) + // Upload apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3060,7 +3015,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster + * Upload big-size app to GCS for Indexer Cluster and Search Head Cluster * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready * When app download is complete, delete apps from app directory @@ -3075,28 +3030,28 @@ var _ = Describe("c3appfw test", func() { */ //################## SETUP #################### - // Download big size apps from Gcs + // Download big size apps from GCS appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") - // Upload big size app to Gcs for Indexer Cluster + // Upload big size app to GCS for Indexer Cluster appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") + testcaseEnvInst.Log.Info("Upload big size app to GCS for Indexer Cluster") gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") + // Upload big size app to GCS for Search Head Cluster + testcaseEnvInst.Log.Info("Upload big size app to GCS for Search Head Cluster") gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") + Expect(err).To(Succeed(), "Unable to upload big size to GCS test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3158,7 +3113,7 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster + * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster * Prepare and deploy C3 CRD with app framework * Verify IsDeploymentInProgress is set * Wait for the pods to be ready @@ -3168,21 +3123,21 @@ var _ = Describe("c3appfw test", func() { appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) + // Upload V1 apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) + // Upload V1 apps to GCS for Search Head Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -3218,7 +3173,7 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { /* Test Steps @@ -3279,12 +3234,12 @@ var _ = Describe("c3appfw test", func() { }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps ################## SETUP #################### - * Upload ES app to Gcs + * Upload ES app to GCS * Upload TA add-on app to location for Indexer cluster * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA * Prepare and deploy C3 CRD with app framework and wait for pods to be ready @@ -3292,43 +3247,43 @@ var _ = Describe("c3appfw test", func() { * Verify ES app is installed on Deployer and on Search Heads * Verify TA add-on app is installed on indexers ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location + * Update ES app on GCS location * Verify updated ES app is installed on Deployer and on Search Heads */ //################## SETUP #################### - // Download ES app from Gcs + // Download ES app from GCS appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") + testcaseEnvInst.Log.Info("Download ES app from GCS") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") + // Download Technology add-on app from GCS + testcaseEnvInst.Log.Info("Download Technology add-on app from GCS") taApp := []string{"Splunk_TA_ForIndexers"} appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileListIdxc) + Expect(err).To(Succeed(), "Unable to download ES app file from GCS") - // Create directory for file upload to Gcs + // Create directory for file upload to GCS gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") + // Upload ES app to GCS + testcaseEnvInst.Log.Info("Upload ES app to GCS") uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") + Expect(err).To(Succeed(), "Unable to upload ES app to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) + // Upload Technology add-on apps to GCS for Indexer Cluster + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to GCS for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for SHC + // Create App Framework Spec for SHC appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) @@ -3339,7 +3294,7 @@ var _ = Describe("c3appfw test", func() { }, } - // Create App framework Spec for Indexer Cluster + // Create App Framework Spec for Indexer Cluster appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) @@ -3414,16 +3369,16 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // //############### UPGRADE APPS ################ - // // Download ES App from Gcs + // // Download ES App from GCS // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") + // testcaseEnvInst.Log.Info("Download updated ES app from GCS") // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) // Expect(err).To(Succeed(), "Unable to download ES app") - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) + // // Upload V2 ES app to GCS for Search Head Cluster + // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to GCS for Search Head Cluster", appVersion)) // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) + // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to GCS test directory for Search Head Cluster", appVersion)) // uploadedApps = append(uploadedApps, uploadedFiles...) // // Check for changes in App phase to determine if next poll has been triggered diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go index 8f4a28249..b2c039d50 100644 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) @@ -52,52 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go index 96759b180..b3d7f39b4 100644 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ b/test/appframework_gcp/m4/appframework_gcs_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -53,16 +52,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -71,30 +61,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -135,7 +105,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -171,7 +141,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -187,19 +157,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -270,17 +231,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -313,7 +265,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -354,7 +306,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -390,7 +342,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -403,19 +355,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -472,17 +415,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -512,7 +446,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -567,7 +501,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -582,17 +516,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -634,9 +559,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -654,8 +577,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -724,9 +646,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -742,8 +662,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -791,7 +710,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -833,7 +752,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -848,14 +767,8 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -897,14 +810,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs = testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -924,7 +831,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -968,7 +875,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) @@ -1004,7 +911,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -1015,19 +922,10 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1079,17 +977,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1182,7 +1071,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1228,7 +1117,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -1242,14 +1131,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -1291,14 +1174,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ appVersion = "V1" @@ -1365,7 +1242,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1431,7 +1308,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1441,7 +1318,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster master with scope local and append cluster scope + // Create App Framework Spec for Cluster Master with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1453,7 +1330,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} @@ -1473,17 +1350,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1553,17 +1421,8 @@ var _ = Describe("m4appfw test", func() { // Check for changes in App phase to determine if next poll has been triggered testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1651,7 +1510,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1670,7 +1529,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1683,7 +1542,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1788,7 +1647,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Master @@ -1807,7 +1666,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1821,7 +1680,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1870,7 +1729,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1892,7 +1751,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1908,7 +1767,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -1920,7 +1779,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1928,17 +1787,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -1962,7 +1812,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1984,7 +1834,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -2000,7 +1850,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2012,7 +1862,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2020,17 +1870,8 @@ var _ = Describe("m4appfw test", func() { // Delete Operator pod while Install in progress testenv.DeleteOperatorPod(testcaseEnvInst) - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2054,7 +1895,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2090,7 +1931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2102,19 +1943,10 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2166,7 +1998,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2208,7 +2040,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) @@ -2222,14 +2054,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2256,8 +2082,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Master CR with latest config cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Master") // Set AppsRepoPollInterval for Cluster Master to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") @@ -2267,8 +2092,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2351,7 +2175,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2383,7 +2207,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2398,7 +2222,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) @@ -2410,7 +2234,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2470,7 +2294,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2499,7 +2323,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2510,7 +2334,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2519,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2537,14 +2361,8 @@ var _ = Describe("m4appfw test", func() { cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 indexers and SHC are ready + testcaseEnvInst.VerifyM4IndexersAndSHCReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Get Pod age to check for pod resets later splunkPodUIDs := testenv.GetPodUIDs(testcaseEnvInst.GetName()) @@ -2562,7 +2380,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2584,7 +2402,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2600,7 +2418,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2612,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Master testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2623,17 +2441,8 @@ var _ = Describe("m4appfw test", func() { err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) Expect(err).To(Succeed(), "Unable to delete file on pod") - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + // Ensure M4 cluster is ready + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) @@ -2657,7 +2466,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, masterappframeworkm4gcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2684,7 +2493,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) @@ -2696,7 +2505,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") @@ -2709,14 +2518,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, shc.Name, shc.Kind) - // Ensure the Indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterMasterReady) // Verify RF SF is met testcaseEnvInst.VerifyRFSFMet(ctx, deployment) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go index bc6837431..36d3f0f89 100644 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ b/test/appframework_gcp/m4/manager_appframework_test.go @@ -22,7 +22,6 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -52,16 +51,7 @@ var _ = Describe("m4appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) @@ -70,30 +60,10 @@ var _ = Describe("m4appfw test", func() { }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGcsBucket, uploadedApps), filePresentOnOperator) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps @@ -134,7 +104,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -170,7 +140,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -186,7 +156,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -312,7 +282,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps @@ -353,7 +323,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) @@ -389,7 +359,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -402,7 +372,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -511,7 +481,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps @@ -566,7 +536,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -633,9 +603,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING UP ################ // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas @@ -653,8 +621,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Indexer CR with latest config idxcName := deployment.GetName() + "-" + "site1" idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -723,9 +690,7 @@ var _ = Describe("m4appfw test", func() { //############### SCALING DOWN ############## // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to get instance of Search Head Cluster") // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas @@ -741,8 +706,7 @@ var _ = Describe("m4appfw test", func() { testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingDown) // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") defaultIndexerReplicas = idxc.Spec.Replicas scaledIndexerReplicas = defaultIndexerReplicas - 1 testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) @@ -790,7 +754,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps @@ -832,7 +796,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -923,7 +887,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { /* Test Steps ################## SETUP #################### @@ -967,7 +931,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) @@ -1003,7 +967,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1014,7 +978,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -1181,7 +1145,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { /* Test Steps @@ -1227,7 +1191,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -1364,7 +1328,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { /* Test Steps @@ -1430,7 +1394,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) @@ -1440,7 +1404,7 @@ var _ = Describe("m4appfw test", func() { appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - // Create App framework Spec for Cluster manager with scope local and append cluster scope + // Create App Framework Spec for Cluster manager with scope local and append cluster scope appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} @@ -1452,7 +1416,7 @@ var _ = Describe("m4appfw test", func() { appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - // Create App framework Spec for Search head cluster with scope local and append cluster scope + // Create App Framework Spec for Search Head cluster with scope local and append cluster scope appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} @@ -1650,7 +1614,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1669,7 +1633,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1682,7 +1646,7 @@ var _ = Describe("m4appfw test", func() { siteCount := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1786,7 +1750,7 @@ var _ = Describe("m4appfw test", func() { // Download all test apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCP for Cluster Manager @@ -1805,7 +1769,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -1819,7 +1783,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1868,7 +1832,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { /* Test Steps @@ -1890,7 +1854,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1906,7 +1870,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -1918,7 +1882,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App installation is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) @@ -1960,7 +1924,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { /* Test Steps @@ -1982,7 +1946,7 @@ var _ = Describe("m4appfw test", func() { // Download all apps from GCP appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload V1 apps to GCP for Indexer Cluster @@ -1998,7 +1962,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2010,7 +1974,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -2052,7 +2016,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { /* Test Steps @@ -2088,7 +2052,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2100,7 +2064,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Ensure that the Cluster Manager goes to Ready phase testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) @@ -2164,7 +2128,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { /* Test Steps @@ -2206,7 +2170,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) @@ -2254,8 +2218,7 @@ var _ = Describe("m4appfw test", func() { //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ // Get instance of current Cluster Manager CR with latest config cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") + testenv.GetInstanceWithExpect(ctx, deployment, cm, deployment.GetName(), "Failed to edit Cluster Manager") // Set AppsRepoPollInterval for Cluster Manager to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") @@ -2265,8 +2228,7 @@ var _ = Describe("m4appfw test", func() { // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Failed to edit Search Head Cluster") // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") @@ -2349,7 +2311,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { /* Test Steps @@ -2381,7 +2343,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload V1 apps to GCP for Indexer Cluster @@ -2396,7 +2358,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) @@ -2408,7 +2370,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is in progress on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2468,7 +2430,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("m4, integration, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { /* Test Steps @@ -2497,7 +2459,7 @@ var _ = Describe("m4appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCP for Indexer Cluster @@ -2508,7 +2470,7 @@ var _ = Describe("m4appfw test", func() { uploadedApps = append(uploadedApps, uploadedFiles...) // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) + testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) @@ -2517,7 +2479,7 @@ var _ = Describe("m4appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 30 - // Create App framework Spec for C3 + // Create App Framework Spec for C3 appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -2560,7 +2522,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { /* Test Steps @@ -2582,7 +2544,7 @@ var _ = Describe("m4appfw test", func() { // Download big size apps from GCP appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big-size app") // Upload big size app to GCP for Indexer Cluster @@ -2598,7 +2560,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2610,7 +2572,7 @@ var _ = Describe("m4appfw test", func() { shReplicas := 3 indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify App Download is completed on Cluster Manager testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -2655,7 +2617,7 @@ var _ = Describe("m4appfw test", func() { }) }) - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { It(" m4gcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { /* Test Steps @@ -2682,7 +2644,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for M4 + // Create App Framework Spec for M4 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) @@ -2694,7 +2656,7 @@ var _ = Describe("m4appfw test", func() { indexersPerSite := 1 cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go index af2fab4c2..90a237aa0 100644 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_suite_test.go @@ -17,7 +17,6 @@ import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +24,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) @@ -41,9 +31,6 @@ var ( appListV2 []string testDataGcsBucket = os.Getenv("TEST_BUCKET") testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation currDir, _ = os.Getwd() downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) @@ -52,47 +39,15 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - + testenvInstance, appListV1, appListV2 = testenv.SetupGCPAppsSuite(testSuiteName, testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, testenv.AppLocationV2, downloadDirV2) }) var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") + testenv.CleanupLocalAppDownloads(testenvInstance, downloadDirV1, downloadDirV2) }) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go index edfd8019f..bd5346e7d 100644 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ b/test/appframework_gcp/s1/appframework_gcs_test.go @@ -23,7 +23,6 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -46,43 +45,14 @@ var _ = Describe("s1appfw test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } + testenv.TeardownAppFrameworkTestCaseEnv(ctx, testcaseEnvInst, deployment, testenv.GCPCloudCleanup(testGCSBucket, uploadedApps), filePresentOnOperator) }) Context("Standalone deployment (S1) with App Framework", func() { @@ -130,7 +100,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -164,7 +134,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -185,7 +155,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -310,7 +280,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -334,7 +304,7 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -354,7 +324,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -481,7 +451,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Monitoring Console + // Create App Framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -511,7 +481,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -531,7 +501,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -566,8 +536,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -599,8 +568,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(ctx, standalone) @@ -669,7 +637,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App Framework Spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -686,7 +654,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -706,8 +674,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Scale up Standalone") standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") standalone.Spec.Replicas = int32(scaledReplicaCount) @@ -773,7 +740,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Download ES app from gcs") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload ES app to gcs @@ -782,7 +749,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ @@ -805,7 +772,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone with App Framework") // Ensure Standalone goes to Ready phase testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -829,7 +796,7 @@ var _ = Describe("s1appfw test", func() { // Download ES App from gcs testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV2, downloadDirV2, appFileList) Expect(err).To(Succeed(), "Unable to download ES app") // Upload V2 apps to gcs for Standalone @@ -884,7 +851,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from gcs testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -894,7 +861,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -974,7 +941,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) @@ -1003,7 +970,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) @@ -1023,7 +990,7 @@ var _ = Describe("s1appfw test", func() { // Create Standalone Deployment with App Framework standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1136,7 +1103,7 @@ var _ = Describe("s1appfw test", func() { // Download apps from GCS testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to GCS for first Standalone @@ -1154,7 +1121,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App Framework Spec appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1168,7 +1135,7 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create App framework Spec + // Create App Framework Spec appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) @@ -1238,7 +1205,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -1266,7 +1233,7 @@ var _ = Describe("s1appfw test", func() { // Download all test apps from GCS appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1277,7 +1244,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1297,7 +1264,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App installation is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) @@ -1345,7 +1312,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1354,7 +1321,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1371,7 +1338,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // ############ Verify livenessProbe and readinessProbe config object and scripts############ testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") @@ -1435,7 +1402,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := append(testenv.BigSingleApp, testenv.ExtraApps...) appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload big-size app to GCS for Standalone @@ -1444,7 +1411,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1461,7 +1428,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) @@ -1517,7 +1484,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1535,7 +1502,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) @@ -1597,7 +1564,7 @@ var _ = Describe("s1appfw test", func() { * Verify no pod resets triggered due to app install * Verify App enabled and version by running splunk cmd // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec + * Create App Framework volume with random credentials and apply to Spec * Check for changes in App phase to determine if next poll has been triggered ############ UPGRADE V2 APPS ########### * Upload V2 apps to GCS App Source @@ -1628,7 +1595,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1647,7 +1614,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) Expect(err).To(Succeed(), "Unable to obtain secret object") @@ -1667,7 +1634,7 @@ var _ = Describe("s1appfw test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec + // Create App Framework volume with invalid credentials and apply to Spec testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) Expect(err).To(Succeed(), "Unable to update secret Object") @@ -1740,7 +1707,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appListV1 := []string{appListV1[0]} appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps") // Upload apps to GCS for Standalone @@ -1749,7 +1716,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) spec := enterpriseApi.StandaloneSpec{ @@ -1766,7 +1733,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App download is in progress on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1832,7 +1799,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.PVTestApps appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) + err = testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.PVTestAppsLocation, downloadDirPVTestApps, appFileList) Expect(err).To(Succeed(), "Unable to download app files") // Upload apps to GCS @@ -1844,7 +1811,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 15 - // Create App framework Spec + // Create App Framework Spec appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -1899,7 +1866,7 @@ var _ = Describe("s1appfw test", func() { appVersion := "V1" appList := testenv.BigSingleApp appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) + err := testenv.DownloadFilesFromGCP(testDataGcsBucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download big app") // Upload big-size app to GCS for Standalone @@ -1908,7 +1875,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) spec := enterpriseApi.StandaloneSpec{ @@ -1925,7 +1892,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify App Download is completed on Standalone testcaseEnvInst.VerifyAppState(ctx, deployment, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) @@ -1979,7 +1946,7 @@ var _ = Describe("s1appfw test", func() { Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework spec for Monitoring Console + // Create App Framework spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) @@ -2017,7 +1984,7 @@ var _ = Describe("s1appfw test", func() { // Maximum apps to be downloaded in parallel maxConcurrentAppDownloads := 5 - // Create App framework spec for Standalone + // Create App Framework spec for Standalone appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) @@ -2038,7 +2005,7 @@ var _ = Describe("s1appfw test", func() { // Deploy Standalone testcaseEnvInst.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") // Verify IsDeploymentInProgress Flag is set to true for Standalone CR testcaseEnvInst.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, deployment.GetName(), standalone.Kind) diff --git a/test/custom_resource_crud/crud_test_shared.go b/test/custom_resource_crud/crud_test_shared.go new file mode 100644 index 000000000..e7a625e91 --- /dev/null +++ b/test/custom_resource_crud/crud_test_shared.go @@ -0,0 +1,283 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// RunS1CPUUpdateTest runs the standard S1 CPU limit update test workflow +func RunS1CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string, newCPULimits string) { + // Deploy and verify Standalone + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), deployment.GetName(), "") + + // Verify telemetry + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Verify CPU limits before updating the CR + standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) + + // Change CPU limits to trigger CR update + standalone.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + err := deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") + + // Verify Standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Verify Standalone goes to ready state + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) +} + +// RunC3CPUUpdateTest runs the standard C3 CPU limit update test workflow +func RunC3CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready + config.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst) + + // Verify telemetry + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console, RF/SF + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + testcaseEnvInst.StandardC3Verification(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits on Indexers before updating the CR + indexerCount := 3 + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to get instance of Indexer Cluster") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-idxc" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Verify CPU limits on Indexers after updating the CR + testcaseEnvInst.VerifyIndexerCPULimits(deployment, deployment.GetName(), indexerCount, newCPULimits) + + // Verify CPU limits on Search Heads before updating the CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + shc := &enterpriseApi.SearchHeadCluster{} + instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) + testenv.GetInstanceWithExpect(ctx, deployment, shc, instanceName, "Unable to fetch Search Head Cluster deployment") + + shc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head Cluster is updating + testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify CPU limits on Search Heads after updating the CR + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, newCPULimits) +} + +// RunC3PVCDeletionTest runs the standard C3 PVC deletion test workflow +func RunC3PVCDeletionTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, verificationTimeout time.Duration) { + // Deploy Single site Cluster and Search Head Clusters + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Verify cluster is ready and RF/SF is met + config.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + + clusterManagerType := config.ClusterManagerPVCType() + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, true, verificationTimeout) + + // Delete the Search Head Cluster + shc := &enterpriseApi.SearchHeadCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, shc, deployment.GetName()+"-shc", "Unable to GET SHC instance") + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) + + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, idxc, deployment.GetName()+"-idxc", "Unable to GET IDXC instance") + err = deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) + + // Delete the Cluster Manager (v3 or v4) + config.DeleteClusterManager(ctx, deployment) + + // Delete Monitoring Console + testenv.GetInstanceWithExpect(ctx, deployment, mc, mcRef, "Unable to GET Monitoring Console instance") + err = deployment.DeleteCR(ctx, mc) + Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) + + verifyC3ClusterPVCs(testcaseEnvInst, deployment, clusterManagerType, false, verificationTimeout) + + // Verify Monitoring Console PVCs (etc and var) have been deleted + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) +} + +// verifyC3ClusterPVCs verifies that PVCs for SHC, deployer, indexers, and cluster manager exist or are deleted. +func verifyC3ClusterPVCs(testcaseEnvInst *testenv.TestCaseEnv, deployment *testenv.Deployment, clusterManagerType string, exists bool, timeout time.Duration) { + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, exists, timeout) + testcaseEnvInst.VerifyPVCsPerDeployment(deployment, clusterManagerType, 1, exists, timeout) +} + +// RunSHCDeployerResourceSpecTest deploys a Search Head Cluster, verifies default CPU limits, +// updates the deployer resource spec, and verifies the deployer is reconfigured while search heads retain defaults. +func RunSHCDeployerResourceSpecTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, defaultCPULimits string) { + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "shc", shcName) + + // Verify CPU limits on Search Heads and deployer before updating CR + searchHeadCount := 3 + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + deployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) + testcaseEnvInst.VerifyCPULimits(deployment, deployerPodName, defaultCPULimits) + + shc := &enterpriseApi.SearchHeadCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to fetch Search Head Cluster deployment") + + // Assign new resources for deployer pod only + newCPULimits := "4" + newCPURequests := "2" + newMemoryLimits := "14Gi" + newMemoryRequests := "12Gi" + + depResSpec := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse(newCPURequests), + "memory": resource.MustParse(newMemoryRequests), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + "memory": resource.MustParse(newMemoryLimits), + }, + } + shc.Spec.DeployerResourceSpec = depResSpec + + testenv.UpdateCRWithExpect(ctx, deployment, shc, "Unable to deploy Search Head Cluster with updated CR") + + // Verify Search Head go to ready state + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Verify CPU limits on Search Heads - Should be same as before + testcaseEnvInst.VerifySearchHeadCPULimits(deployment, deployment.GetName(), searchHeadCount, defaultCPULimits) + + // Verify modified deployer spec + testcaseEnvInst.VerifyResourceConstraints(deployment, deployerPodName, depResSpec) +} + +// RunM4CPUUpdateTest runs the standard M4 CPU limit update test workflow +func RunM4CPUUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, defaultCPULimits string, newCPULimits string) { + // Deploy Multisite Cluster and Search Head Clusters + mcRef := deployment.GetName() + prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) + siteCount := 3 + err := config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure that the cluster-manager goes to Ready phase + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + testcaseEnvInst.TriggerAndVerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, "") + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits on Indexers before updating the CR + testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, deployment.GetName(), siteCount, defaultCPULimits) + + // Change CPU limits to trigger CR update + idxc := &enterpriseApi.IndexerCluster{} + for i := 1; i <= siteCount; i++ { + siteName := fmt.Sprintf("site%d", i) + instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) + testenv.GetInstanceWithExpect(ctx, deployment, idxc, instanceName, "Unable to fetch Indexer Cluster deployment") + idxc.Spec.Resources.Limits = corev1.ResourceList{ + "cpu": resource.MustParse(newCPULimits), + } + testenv.UpdateCRWithExpect(ctx, deployment, idxc, "Unable to deploy Indexer Cluster with updated CR") + } + + // Verify Indexer Cluster is updating + idxcName := deployment.GetName() + "-site1" + testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) + + // Verify Indexers go to ready state + testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // Verify RF SF is met + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + // Verify CPU limits after updating the CR + testcaseEnvInst.VerifyCPULimitsOnAllSites(deployment, deployment.GetName(), siteCount, newCPULimits) +} diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go deleted file mode 100644 index 8c47e63b2..000000000 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Master - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Master PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, splcommon.ClusterManager, 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_m4_test.go b/test/custom_resource_crud/custom_resource_crud_m4_test.go deleted file mode 100644 index 4c00a07dc..000000000 --- a/test/custom_resource_crud/custom_resource_crud_m4_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_s1_test.go b/test/custom_resource_crud/custom_resource_crud_s1_test.go deleted file mode 100644 index deb487628..000000000 --- a/test/custom_resource_crud/custom_resource_crud_s1_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1)", func() { - It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { - - // Deploy Standalone - mcRef := deployment.GetName() - prevTelemetrySubmissionTime := testcaseEnvInst.GetTelemetryLastSubmissionTime(ctx, deployment) - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify telemetry - testcaseEnvInst.TriggerTelemetrySubmission(ctx, deployment) - testcaseEnvInst.VerifyTelemetry(ctx, deployment, prevTelemetrySubmissionTime) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits before updating the CR - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, defaultCPULimits) - - // Change CPU limits to trigger CR update - standalone.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with updated CR ") - - // Verify Standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify Standalone goes to ready state - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits after updating the CR - testcaseEnvInst.VerifyCPULimits(deployment, standalonePodName, newCPULimits) - }) - }) -}) diff --git a/test/custom_resource_crud/custom_resource_crud_suite_test.go b/test/custom_resource_crud/custom_resource_crud_suite_test.go index 8972ac76e..ce753a736 100644 --- a/test/custom_resource_crud/custom_resource_crud_suite_test.go +++ b/test/custom_resource_crud/custom_resource_crud_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,12 +24,12 @@ import ( ) const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond + // DefaultCPULimits is the default CPU limit + DefaultCPULimits = "4" + // UpdatedCPULimits is the updated CPU limit + UpdatedCPULimits = "2" + // DefaultVerificationTimeout is the default timeout for CRUD verification steps + DefaultVerificationTimeout = 150 * time.Second ) var ( @@ -39,7 +39,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/custom_resource_crud/custom_resource_crud_test.go b/test/custom_resource_crud/custom_resource_crud_test.go new file mode 100644 index 000000000..2c5c069d6 --- /dev/null +++ b/test/custom_resource_crud/custom_resource_crud_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package crcrud + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerCrudConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 CRUD test tables. +var masterManagerCrudConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastercrcrud", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managercrcrud", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Custom Resource CRUD test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var defaultCPULimits string + var newCPULimits string + var verificationTimeout time.Duration + + ctx := context.TODO() + + // S1 test — single variant (manager, V4) + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, s1: can deploy a standalone instance, change its CR, update the instance", func() { + RunS1CPUUpdateTest(ctx, deployment, testcaseEnvInst, defaultCPULimits, newCPULimits) + }) + }) + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + verificationTimeout = DefaultVerificationTimeout + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.Label+", integration, c3: can deploy Indexer and Search Head Cluster, change their CR, update the instances", func() { + config := tc.NewConfig() + RunC3CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + + It(tc.Label+", integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { + config := tc.NewConfig() + RunC3PVCDeletionTest(ctx, deployment, testcaseEnvInst, config, verificationTimeout) + }) + }) + } + + // CSPL-3256 - SHC deployer resource spec test (IDXC is irrelevant for this test case) + Context("Search Head Cluster", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { + RunSHCDeployerResourceSpecTest(ctx, deployment, testcaseEnvInst, defaultCPULimits) + }) + }) + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerCrudConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + defaultCPULimits = DefaultCPULimits + newCPULimits = UpdatedCPULimits + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.Label+", integration, m4: can deploy multisite Indexer and Search Head Clusters, change their CR, update the instances", func() { + config := tc.NewConfig() + RunM4CPUUpdateTest(ctx, deployment, testcaseEnvInst, config, defaultCPULimits, newCPULimits) + }) + }) + } +}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go deleted file mode 100644 index 86bcea539..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var verificationTimeout time.Duration - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - verificationTimeout = 150 * time.Second - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can deploy indexer and search head cluster, change their CR, update the instances", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - indexerCount := 3 - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - instanceName := fmt.Sprintf("%s-idxc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to get instance of indexer cluster") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-idxc" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify CPU limits on Indexers after updating the CR - for i := 0; i < indexerCount; i++ { - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, indexerPodName, newCPULimits) - } - - // Verify CPU limits on Search Heads before updating the CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - shc := &enterpriseApi.SearchHeadCluster{} - instanceName = fmt.Sprintf("%s-shc", deployment.GetName()) - err = deployment.GetInstance(ctx, instanceName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - shc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head Cluster is updating - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify CPU limits on Search Heads after updating the CR - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, newCPULimits) - } - }) - }) - - Context("Search Head Cluster", func() { - // CSPL-3256 - Adding the SHC only test case under c3 as IDXC is irrelevant for this test case - It("managercrcrud, integration, shc: can deploy Search Head Cluster with Deployer resource spec configured", func() { - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - _, err := deployment.DeploySearchHeadCluster(ctx, shcName, "", "", "", "") - if err != nil { - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster", "Shc", shcName) - } - - // Verify CPU limits on Search Heads and deployer before updating CR - searchHeadCount := 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - DeployerPodName := fmt.Sprintf(testenv.DeployerPod, deployment.GetName()) - testcaseEnvInst.VerifyCPULimits(deployment, DeployerPodName, defaultCPULimits) - - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Unable to fetch Search Head Cluster deployment") - - // Assign new resources for deployer pod only - newCPULimits = "4" - newCPURequests := "2" - newMemoryLimits := "14Gi" - newMemoryRequests := "12Gi" - - depResSpec := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse(newCPURequests), - "memory": resource.MustParse(newMemoryRequests), - }, - Limits: corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - "memory": resource.MustParse(newMemoryLimits), - }, - } - shc.Spec.DeployerResourceSpec = depResSpec - - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster with updated CR") - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify CPU limits on Search Heads - Should be same as before - searchHeadCount = 3 - for i := 0; i < searchHeadCount; i++ { - SearchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) - testcaseEnvInst.VerifyCPULimits(deployment, SearchHeadPodName, defaultCPULimits) - } - - // Verify modified deployer spec - testcaseEnvInst.VerifyResourceConstraints(deployment, DeployerPodName, depResSpec) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managercrcrud, integration, c3: can verify IDXC, CM and SHC PVCs are correctly deleted after the CRs deletion", func() { - - // Deploy Single site Cluster and Search Head Clusters - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify Search Head go to ready state - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify Search Heads PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, true, verificationTimeout) - - // Verify Deployer PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, true, verificationTimeout) - - // Verify Indexers PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, true, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) exists - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, true, verificationTimeout) - - // Delete the Search Head Cluster - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Unable to GET SHC instance", "SHC Name", shc) - err = deployment.DeleteCR(ctx, shc) - Expect(err).To(Succeed(), "Unable to delete SHC instance", "SHC Name", shc) - - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to GET IDXC instance", "IDXC Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete IDXC instance", "IDXC Name", idxc) - - // Delete the Cluster Manager - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manger Name", cm) - - // Delete Monitoring Console - err = deployment.GetInstance(ctx, mcRef, mc) - Expect(err).To(Succeed(), "Unable to GET Monitoring Console instance", "Monitoring Console Name", mcRef) - err = deployment.DeleteCR(ctx, mc) - Expect(err).To(Succeed(), "Unable to delete Monitoring Console instance", "Monitoring Console Name", mcRef) - - // Verify Search Heads PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-search-head", 3, false, verificationTimeout) - - // Verify Deployer PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "shc-deployer", 1, false, verificationTimeout) - - // Verify Indexers PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "idxc-indexer", 3, false, verificationTimeout) - - // Verify Cluster Manager PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "cluster-manager", 1, false, verificationTimeout) - - // Verify Monitoring Console PVCs (etc and var) have been deleted - testcaseEnvInst.VerifyPVCsPerDeployment(deployment, "monitoring-console", 1, false, verificationTimeout) - }) - }) -}) diff --git a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go b/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go deleted file mode 100644 index 89fadcbb8..000000000 --- a/test/custom_resource_crud/manager_custom_resource_crud_m4_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package crcrud - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Crcrud test for SVA M4", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var defaultCPULimits string - var newCPULimits string - var ctx context.Context - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - ctx = context.TODO() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - - defaultCPULimits = "4" - newCPULimits = "2" - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managercrcrud, integration, m4: can deploy can deploy multisite indexer and search head clusters, change their CR, update the instances", func() { - - // Deploy Multisite Cluster and Search Head Clusters - mcRef := deployment.GetName() - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits on Indexers before updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, defaultCPULimits) - } - - // Change CPU limits to trigger CR update - idxc := &enterpriseApi.IndexerCluster{} - for i := 1; i <= siteCount; i++ { - siteName := fmt.Sprintf("site%d", i) - instanceName := fmt.Sprintf("%s-%s", deployment.GetName(), siteName) - err = deployment.GetInstance(ctx, instanceName, idxc) - Expect(err).To(Succeed(), "Unable to fetch Indexer Cluster deployment") - idxc.Spec.Resources.Limits = corev1.ResourceList{ - "cpu": resource.MustParse(newCPULimits), - } - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with updated CR") - } - - // Verify Indexer Cluster is updating - idxcName := deployment.GetName() + "-" + "site1" - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseUpdating, idxcName) - - // Verify Indexers go to ready state - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify CPU limits after updating the CR - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - testcaseEnvInst.VerifyCPULimits(deployment, podName, newCPULimits) - } - }) - }) -}) diff --git a/test/delete_cr/deletecr_suite_test.go b/test/delete_cr/deletecr_suite_test.go index 49b34af39..83d4a3c11 100644 --- a/test/delete_cr/deletecr_suite_test.go +++ b/test/delete_cr/deletecr_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "deletecr-" + testenv.RandomDNSName(3) @@ -41,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index 2f3667ba1..167d4ea24 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -1,138 +1,49 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package deletecr import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" + "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("DeleteCR test", func() { +var _ = Describe("Delete CR test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) - Context("Standalone deployment (S1 - Standalone Pod)", func() { + Context("Standalone deployment (S1)", func() { It("integration, managerdeletecr: can deploy standalone and delete", func() { - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Delete Standalone CR - err = deployment.DeleteCR(ctx, standalone) - Expect(err).To(Succeed(), "Unable to Delete Standalone") - + testenv.RunDeleteStandaloneWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) Context("Single Site Indexer Cluster with Search Head Cluster (C3)", func() { It("integration, managerdeletecr: can deploy C3 and delete search head, clustermanager", func() { - - // Deploy C3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, true, "") - Expect(err).To(Succeed(), "Unable to deploy C3 instance") - - // Ensure Cluster Manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - idxc := &enterpriseApi.IndexerCluster{} - idxcName := deployment.GetName() + "-idxc" - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Unable to get Indexer instance") - - // Delete Indexer Cluster CR - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to Delete Indexer Cluster") - - sh := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, sh) - Expect(err).To(Succeed(), "Unable to get Search Head instance") - - // Delete Search Head Cluster CR - err = deployment.DeleteCR(ctx, sh) - Expect(err).To(Succeed(), "Unable to Delete Search Head Cluster") - - cm := &enterpriseApi.ClusterManager{} - cmName := deployment.GetName() - err = deployment.GetInstance(ctx, cmName, cm) - Expect(err).To(Succeed(), "Unable to get Cluster Manager instance") - - // Delete Cluster Manager CR - err = deployment.DeleteCR(ctx, cm) - Expect(err).To(Succeed(), "Unable to Delete Cluster Manager") - + testenv.RunDeleteC3Workflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3) }) }) }) diff --git a/test/example/example1_test.go b/test/example/example1_test.go index bba0023d1..37e214084 100644 --- a/test/example/example1_test.go +++ b/test/example/example1_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example1", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example2_test.go b/test/example/example2_test.go index 3988e0976..9ca6b9959 100644 --- a/test/example/example2_test.go +++ b/test/example/example2_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,10 @@ package example import ( - "fmt" "math/rand" "time" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,20 +29,11 @@ var _ = XDescribe("Example2", func() { // This is invoke for each "It" spec below BeforeEach(func() { - var err error - // Create a deployment for this test - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - deployment.Teardown() - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) // "It" spec diff --git a/test/example/example_suite_test.go b/test/example/example_suite_test.go index 78f198f31..2e5710526 100644 --- a/test/example/example_suite_test.go +++ b/test/example/example_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ package example import ( - "math/rand" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,12 +27,7 @@ var ( testSuiteName = "example-" + testenv.RandomDNSName(3) ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - func TestExampleSuite(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) @@ -47,5 +40,7 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + if testenvInstance != nil { + Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) + } }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go index d2c4be9f1..4b2732a69 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_suite_test.go @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "os" "path/filepath" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,18 +25,9 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv - testSuiteName = "indingsep-" + testenv.RandomDNSName(3) + testSuiteName = "idxingsep-" + testenv.RandomDNSName(3) queue = enterpriseApi.QueueSpec{ Provider: "sqs", @@ -100,7 +90,6 @@ var ( appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) s3TestDir = "icappfw-" + testenv.RandomDNSName(4) appListV1 = testenv.BasicApps - s3AppDirV1 = testenv.AppLocationV1 ) // TestBasic is the main entry point @@ -119,7 +108,7 @@ var _ = BeforeSuite(func() { appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps from S3 - err = testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, testenv.AppLocationV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download V1 app files") }) diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 9142c9882..db2169a99 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package indingsep +package indexingestionsep import ( "context" @@ -21,38 +21,23 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/onsi/ginkgo/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("indingsep test", func() { +var _ = Describe("Index and Ingestion Separation test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var cmSpec enterpriseApi.ClusterManagerSpec - ctx := context.TODO() BeforeEach(func() { - var err error - - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") cmSpec = enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ @@ -65,101 +50,22 @@ var _ = Describe("indingsep test", func() { }) AfterEach(func() { - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) - // Delete the Indexer Cluster - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", idxc) - Expect(err).To(Succeed(), "Unable to get Indexer Cluster instance", "Indexer Cluster Name", idxc) - err = deployment.DeleteCR(ctx, idxc) - Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) - - // Delete the Ingestor Cluster - ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Unable to get Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - err = deployment.DeleteCR(ctx, ingest) - Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) - - // Delete the Queue - q = &enterpriseApi.Queue{} - err = deployment.GetInstance(ctx, "queue", q) - Expect(err).To(Succeed(), "Unable to get Queue instance", "Queue Name", q) - err = deployment.DeleteCR(ctx, q) - Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) - - // Delete the ObjectStorage - objStorage = &enterpriseApi.ObjectStorage{} - err = deployment.GetInstance(ctx, "os", objStorage) - Expect(err).To(Succeed(), "Unable to get ObjectStorage instance", "ObjectStorage Name", objStorage) - err = deployment.DeleteCR(ctx, objStorage) - Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) + deleteIngestorStack(ctx, deployment) }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, smoke, indingsep: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { + It("indexingestionsep, smoke: Splunk Operator can deploy Ingestors and Indexers with additional configurations", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") @@ -172,15 +78,8 @@ var _ = Describe("indingsep test", func() { )} queue.SQS.VolList = volumeSpec - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + // Deploy Queue and ObjectStorage + q, objStorage := deployQueueAndObjectStorage(ctx, deployment, queue, objectStorage) // Deploy Ingestor Cluster with additional configurations (similar to standalone app framework test) appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) @@ -228,7 +127,7 @@ var _ = Describe("indingsep test", func() { } testcaseEnvInst.Log.Info("Deploy Ingestor Cluster with additional configurations") - _, err = deployment.DeployIngestorClusterWithAdditionalConfiguration(ctx, ic) + _, err := deployment.DeployIngestorClusterWithAdditionalConfiguration(ctx, ic) Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") // Ensure that Ingestor Cluster is in Ready phase @@ -260,73 +159,21 @@ var _ = Describe("indingsep test", func() { testcaseEnvInst.VerifyAppFrameworkState(ctx, deployment, allAppSourceInfo, splunkPodUIDs, "") // Verify probe configuration - testcaseEnvInst.Log.Info("Get config map for probes") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for probes", "ConfigMap", ConfigMapName) - testcaseEnvInst.Log.Info("Verify probe configurations on Ingestor pods") - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, true) }) - }) - Context("Ingestor and Indexer deployment", func() { - It("indingsep, integration, indingsep: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { + It("indexingestionsep, integration: Splunk Operator can deploy Ingestors and Indexers with correct setup", func() { // TODO: Remove secret reference and uncomment serviceAccountName part once IRSA fixed for Splunk and EKS 1.34+ // Create Service Account // testcaseEnvInst.Log.Info("Create Service Account") // testcaseEnvInst.CreateServiceAccount(serviceAccountName) - // Secret reference - volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( - "queue-secret-ref-volume", - testcaseEnvInst.GetIndexIngestSepSecretName(), - )} - queue.SQS.VolList = volumeSpec - - // Deploy Queue - testcaseEnvInst.Log.Info("Deploy Queue") - q, err := deployment.DeployQueue(ctx, "queue", queue) - Expect(err).To(Succeed(), "Unable to deploy Queue") - - // Deploy ObjectStorage - testcaseEnvInst.Log.Info("Deploy ObjectStorage") - objStorage, err := deployment.DeployObjectStorage(ctx, "os", objectStorage) - Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") - - // Deploy Ingestor Cluster - testcaseEnvInst.Log.Info("Deploy Ingestor Cluster") - _, err = deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") - - // Deploy Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Indexer Cluster") - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") // , serviceAccountName) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") - - // Ensure that Ingestor Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Ingestor Cluster is in Ready phase") - testcaseEnvInst.VerifyIngestorReady(ctx, deployment) - - // Ensure that Cluster Manager is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Cluster Manager is in Ready phase") - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure that Indexer Cluster is in Ready phase - testcaseEnvInst.Log.Info("Ensure that Indexer Cluster is in Ready phase") - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + setupIngestorStack(ctx, deployment, testcaseEnvInst, queue, objectStorage, cmSpec) // Get instance of current Ingestor Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Ingestor Cluster CR with latest config") ingest := &enterpriseApi.IngestorCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-ingest", ingest) - Expect(err).To(Succeed(), "Failed to get instance of Ingestor Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, ingest, deployment.GetName()+"-ingest", "Failed to get instance of Ingestor Cluster") // Verify Ingestor Cluster Status testcaseEnvInst.Log.Info("Verify Ingestor Cluster Status") @@ -336,8 +183,7 @@ var _ = Describe("indingsep test", func() { // Get instance of current Indexer Cluster CR with latest config testcaseEnvInst.Log.Info("Get instance of current Indexer Cluster CR with latest config") index := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-idxc", index) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") + testenv.GetInstanceWithExpect(ctx, deployment, index, deployment.GetName()+"-idxc", "Failed to get instance of Indexer Cluster") // Verify Indexer Cluster Status testcaseEnvInst.Log.Info("Verify Indexer Cluster Status") @@ -352,18 +198,10 @@ var _ = Describe("indingsep test", func() { if strings.Contains(pod, "ingest") || strings.Contains(pod, "idxc") { // Verify outputs.conf - testcaseEnvInst.Log.Info("Verify outputs.conf") - outputsPath := "opt/splunk/etc/system/local/outputs.conf" - outputsConf, err := testenv.GetConfFile(pod, outputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get outputs.conf from Ingestor Cluster pod") - testenv.ValidateContent(outputsConf, outputs, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/outputs.conf", deployment.GetName(), outputs, "Failed to get outputs.conf from Ingestor Cluster pod") // Verify default-mode.conf - testcaseEnvInst.Log.Info("Verify default-mode.conf") - defaultsPath := "opt/splunk/etc/system/local/default-mode.conf" - defaultsConf, err := testenv.GetConfFile(pod, defaultsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get default-mode.conf from Ingestor Cluster pod") - testenv.ValidateContent(defaultsConf, defaultsAll, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/default-mode.conf", deployment.GetName(), defaultsAll, "Failed to get default-mode.conf from Ingestor Cluster pod") // Verify AWS env variables testcaseEnvInst.Log.Info("Verify AWS env variables") @@ -378,13 +216,79 @@ var _ = Describe("indingsep test", func() { testenv.ValidateContent(defaultsConf, defaultsIngest, true) } else if strings.Contains(pod, "idxc") { // Verify inputs.conf - testcaseEnvInst.Log.Info("Verify inputs.conf") - inputsPath := "opt/splunk/etc/system/local/inputs.conf" - inputsConf, err := testenv.GetConfFile(pod, inputsPath, deployment.GetName()) - Expect(err).To(Succeed(), "Failed to get inputs.conf from Indexer Cluster pod") - testenv.ValidateContent(inputsConf, inputs, true) + verifyConfFileContent(pod, "opt/splunk/etc/system/local/inputs.conf", deployment.GetName(), inputs, "Failed to get inputs.conf from Indexer Cluster pod") } } }) }) }) + +// verifyConfFileContent retrieves a conf file from a pod and validates its content. +func verifyConfFileContent(pod, confPath, deploymentName string, expectedContent []string, errorMsg string) { + conf, err := testenv.GetConfFile(pod, confPath, deploymentName) + Expect(err).To(Succeed(), errorMsg) + testenv.ValidateContent(conf, expectedContent, true) +} + +// deployQueueAndObjectStorage deploys a Queue and ObjectStorage CR and returns both. +func deployQueueAndObjectStorage(ctx context.Context, deployment *testenv.Deployment, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec) (*enterpriseApi.Queue, *enterpriseApi.ObjectStorage) { + q, err := deployment.DeployQueue(ctx, "queue", qSpec) + Expect(err).To(Succeed(), "Unable to deploy Queue") + + objStorage, err := deployment.DeployObjectStorage(ctx, "os", osSpec) + Expect(err).To(Succeed(), "Unable to deploy ObjectStorage") + + return q, objStorage +} + +// setupIngestorStack deploys the full Queue/ObjectStorage/IngestorCluster/ClusterManager/IndexerCluster stack +// and verifies each component reaches the Ready phase. +func setupIngestorStack(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, qSpec enterpriseApi.QueueSpec, osSpec enterpriseApi.ObjectStorageSpec, cmSpec enterpriseApi.ClusterManagerSpec) { + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} + qSpec.SQS.VolList = volumeSpec + + q, objStorage := deployQueueAndObjectStorage(ctx, deployment, qSpec, osSpec) + + _, err := deployment.DeployIngestorCluster(ctx, deployment.GetName()+"-ingest", 3, v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Ingestor Cluster") + + _, err = deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + + _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", 3, deployment.GetName(), "", v1.ObjectReference{Name: q.Name}, v1.ObjectReference{Name: objStorage.Name}, "") + Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster") + + testcaseEnvInst.VerifyIngestorReady(ctx, deployment) + testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// deleteIngestorStack tears down the full Queue/ObjectStorage/IngestorCluster/IndexerCluster stack. +func deleteIngestorStack(ctx context.Context, deployment *testenv.Deployment) { + // Delete the Indexer Cluster + idxc := &enterpriseApi.IndexerCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, idxc, deployment.GetName()+"-idxc", "Unable to get Indexer Cluster instance") + err := deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster instance", "Indexer Cluster Name", idxc) + + // Delete the Ingestor Cluster + ingest := &enterpriseApi.IngestorCluster{} + testenv.GetInstanceWithExpect(ctx, deployment, ingest, deployment.GetName()+"-ingest", "Unable to get Ingestor Cluster instance") + err = deployment.DeleteCR(ctx, ingest) + Expect(err).To(Succeed(), "Unable to delete Ingestor Cluster instance", "Ingestor Cluster Name", ingest) + + // Delete the Queue + q := &enterpriseApi.Queue{} + testenv.GetInstanceWithExpect(ctx, deployment, q, "queue", "Unable to get Queue instance") + err = deployment.DeleteCR(ctx, q) + Expect(err).To(Succeed(), "Unable to delete Queue", "Queue Name", q) + + // Delete the ObjectStorage + objStorage := &enterpriseApi.ObjectStorage{} + testenv.GetInstanceWithExpect(ctx, deployment, objStorage, "os", "Unable to get ObjectStorage instance") + err = deployment.DeleteCR(ctx, objStorage) + Expect(err).To(Succeed(), "Unable to delete ObjectStorage", "ObjectStorage Name", objStorage) +} diff --git a/test/ingest_search/ingest_search_suite_test.go b/test/ingest_search/ingest_search_suite_test.go index 301f11611..6bce14483 100644 --- a/test/ingest_search/ingest_search_suite_test.go +++ b/test/ingest_search/ingest_search_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package ingestsearchtest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "ingest-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/ingest_search/ingest_search_test.go b/test/ingest_search/ingest_search_test.go index 090f034a9..4fdb8d888 100644 --- a/test/ingest_search/ingest_search_test.go +++ b/test/ingest_search/ingest_search_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,253 +14,34 @@ package ingestsearchtest import ( - "bufio" "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) -var _ = Describe("Ingest and Search Test", func() { +var _ = Describe("Ingest and Search test", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment - var firstLine string ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can search internal logs for standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - searchString := "index=_internal | stats count by host" - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) - - var searchResults map[string]interface{} - unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - if unmarshalErr != nil { - testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") - } - - prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - searchString := "index=_internal GUID component=ServerConfig" - - // Perform a simple search - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) - if reqErr != nil { - testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - if statusErr != nil { - testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - if resErr != nil { - testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) - return enterpriseApi.PhaseError - } - - // Display results for debug purposes - prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") - if jsonErr != nil { - testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") - } else { - testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) - } - - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + RunS1InternalLogSearchTest(ctx, deployment, testcaseEnvInst) }) - }) - Context("Standalone deployment (S1)", func() { It("ingest_search, integration, s1: can ingest custom data to new index and search", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify splunk status is up - Eventually(func() enterpriseApi.Phase { - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - - splunkBin := "/opt/splunk/bin/splunk" - username := "admin" - password := "$(cat /mnt/splunk-secrets/password)" - splunkCmd := "status" - - statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) - command := []string{"/bin/bash"} - statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) - if err != nil { - testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) - return enterpriseApi.PhaseError - } - - if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { - testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) - return enterpriseApi.PhaseError - } - - testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) - return standalone.Status.Phase - }, deployment.GetTimeout(), PollInterval).Should(Equal(enterpriseApi.PhaseReady)) - - // Create an index - podName := fmt.Sprintf("splunk-%s-standalone-0", deployment.GetName()) - indexName := "myTestIndex" - - // Create an index on a standalone instance - err = testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed response to add index to splunk") - - // Create a mock logfile to ingest - logFile := "/tmp/test.log" - err = testenv.CreateMockLogfile(logFile, 1) - Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) - - // Copy log file and ingest it - err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) - Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) - - // Read first line to find a search token - var file, openErr = os.Open(logFile) - Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) - - reader := bufio.NewReader(file) - var readErr error - firstLine, readErr = reader.ReadString('\n') - Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) - - tokens := strings.Fields(firstLine) - Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) - - searchToken := tokens[len(tokens)-1] - testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) - - searchString := fmt.Sprintf("index=%s | stats count by host", indexName) - - // Wait for search results to be available instead of fixed sleep - err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) - Expect(err).To(Succeed(), "Timed out waiting for search results") - - searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) - - // Verify result. Should get count 1. result:{count:1} - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) - - hostCount := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) - testHostCnt := strings.Compare(hostCount["count"].(string), "1") - testHostname := strings.Compare(hostCount["host"].(string), podName) - Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) - - searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) - sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) - Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) - testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) - - // Check SID status until done - searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) - Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) - testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) - - // Get SID results - searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) - Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) - - testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) - var searchResults2 testenv.SearchJobResultsResponse - jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - found := false - for key, elem := range searchResults2.Results { - testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) - trimFirstLine := strings.TrimSuffix(firstLine, "\n") - if strings.Compare(elem.Raw, trimFirstLine) == 0 { - testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) - found = true - } - } - Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) + RunS1IngestAndSearchTest(ctx, deployment, testcaseEnvInst) }) }) }) diff --git a/test/ingest_search/ingest_search_test_shared.go b/test/ingest_search/ingest_search_test_shared.go new file mode 100644 index 000000000..441ecaa91 --- /dev/null +++ b/test/ingest_search/ingest_search_test_shared.go @@ -0,0 +1,204 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ingestsearchtest + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// RunS1InternalLogSearchTest deploys a Standalone instance and verifies internal log searches +// using both synchronous and asynchronous search APIs. +func RunS1InternalLogSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "", "") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + + searchString := "index=_internal | stats count by host" + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Performed a search", "searchString", searchString) + + var searchResults map[string]interface{} + unmarshalErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + if unmarshalErr != nil { + testcaseEnvInst.Log.Error(unmarshalErr, "Failed to unmarshal JSON response") + } + + prettyResults, jsonErr := json.MarshalIndent(searchResults, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Sync Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + searchString := "index=_internal GUID component=ServerConfig" + + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString, deployment) + if reqErr != nil { + testcaseEnvInst.Log.Error(reqErr, "Failed to execute search on pod", "pod", podName, "searchString", searchString) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + if statusErr != nil { + testcaseEnvInst.Log.Error(statusErr, "Failed to get search status on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + if resErr != nil { + testcaseEnvInst.Log.Error(resErr, "Failed to get search results on pod", "pod", podName, "sid", sid) + return enterpriseApi.PhaseError + } + + prettyResults, jsonErr := json.MarshalIndent(searchResultsResp, "", " ") + if jsonErr != nil { + testcaseEnvInst.Log.Error(jsonErr, "Failed to generate pretty json") + } else { + testcaseEnvInst.Log.Info("Search results:", "prettyResults", string(prettyResults)) + } + + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) +} + +// RunS1IngestAndSearchTest deploys a Standalone instance, ingests a custom log file into a new +// index, and verifies the ingested data is searchable via both sync and async search APIs. +func RunS1IngestAndSearchTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, deployment.GetName(), "", "") + + Eventually(func() enterpriseApi.Phase { + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + + splunkBin := "/opt/splunk/bin/splunk" + username := "admin" + password := "$(cat /mnt/splunk-secrets/password)" + splunkCmd := "status" + + statusCmd := fmt.Sprintf("%s %s -auth %s:%s", splunkBin, splunkCmd, username, password) + command := []string{"/bin/bash"} + statusCmdResp, stderr, err := deployment.PodExecCommand(ctx, podName, command, statusCmd, false) + if err != nil { + testcaseEnvInst.Log.Error(err, "Failed to execute command on pod", "pod", podName, "statusCmd", statusCmd, "statusCmdResp", statusCmdResp, "stderr", stderr) + return enterpriseApi.PhaseError + } + + if !strings.Contains(strings.ToLower(statusCmdResp), strings.ToLower("splunkd is running")) { + testcaseEnvInst.Log.Error(err, "Failed to find splunkd running", "pod", podName, "statusCmdResp", statusCmdResp) + return enterpriseApi.PhaseError + } + + testcaseEnvInst.Log.Info("Waiting for standalone splunkd status to be ready", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) + return standalone.Status.Phase + }, deployment.GetTimeout(), testenv.PollInterval).Should(Equal(enterpriseApi.PhaseReady)) + + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + indexName := "myTestIndex" + + err := testenv.CreateAnIndexStandalone(ctx, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed response to add index to splunk") + + logFile := "/tmp/test.log" + err = testenv.CreateMockLogfile(logFile, 1) + Expect(err).To(Succeed(), "Failed response to add index to splunk logfile %s", logFile) + + err = testenv.IngestFileViaOneshot(ctx, logFile, indexName, podName, deployment) + Expect(err).To(Succeed(), "Failed to ingest logfile %s on pod %s", logFile, podName) + + file, openErr := os.Open(logFile) + Expect(openErr).To(Succeed(), "Failed to open newly created logfile %s on pod %s", logFile, podName) + + reader := bufio.NewReader(file) + firstLine, readErr := reader.ReadString('\n') + Expect(readErr).Should(Or(BeNil(), Equal(io.EOF)), "Failed to read first line of logfile %s on pod ", logFile, podName) + + tokens := strings.Fields(firstLine) + Expect(len(tokens)).To(BeNumerically(">=", 2), "Incorrect tokens (%s) in first logline %s for logfile %s", tokens, firstLine, logFile) + + searchToken := tokens[len(tokens)-1] + testcaseEnvInst.Log.Info("Got search token successfully", "logFile", logFile, "searchToken", searchToken) + + searchString := fmt.Sprintf("index=%s | stats count by host", indexName) + + err = testenv.WaitForSearchResultsNonEmpty(ctx, deployment, podName, searchString, 2*time.Second) + Expect(err).To(Succeed(), "Timed out waiting for search results") + + searchResultsResp, err := testenv.PerformSearchSync(ctx, podName, searchString, deployment) + Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", podName, searchString) + + var searchResults map[string]interface{} + jsonErr := json.Unmarshal([]byte(searchResultsResp), &searchResults) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) + Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, podName) + + hostCount := searchResults["result"].(map[string]interface{}) + testcaseEnvInst.Log.Info("Sync Search results host count:", "count", hostCount["count"].(string), "host", hostCount["host"].(string)) + testHostCnt := strings.Compare(hostCount["count"].(string), "1") + testHostname := strings.Compare(hostCount["host"].(string), podName) + Expect(testHostCnt).To(Equal(0), "Incorrect search results for count. Expect: 1 Got: %d", hostCount["count"].(string)) + Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", podName, hostCount["host"].(string)) + + searchString2 := fmt.Sprintf("index=%s %s", indexName, searchToken) + sid, reqErr := testenv.PerformSearchReq(ctx, podName, searchString2, deployment) + Expect(reqErr).To(Succeed(), "Failed to execute search '%s' on pod %s", searchString, podName) + testcaseEnvInst.Log.Info("Got a search with sid", "sid", sid) + + searchStatusResult, statusErr := testenv.GetSearchStatus(ctx, podName, sid, deployment) + Expect(statusErr).To(Succeed(), "Failed to get search status on pod %s for sid %s", podName, sid) + testcaseEnvInst.Log.Info("Search status:", "searchStatusResult", searchStatusResult) + + searchResultsResp, resErr := testenv.GetSearchResults(ctx, podName, sid, deployment) + Expect(resErr).To(Succeed(), "Failed to get search results on pod %s for sid %s", podName, sid) + + testcaseEnvInst.Log.Info("Raw Search results:", "searchResultsResp", searchResultsResp) + var searchResults2 testenv.SearchJobResultsResponse + jsonErr = json.Unmarshal([]byte(searchResultsResp), &searchResults2) + Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) + + found := false + for key, elem := range searchResults2.Results { + testcaseEnvInst.Log.Info("Search results _raw and host:", "_raw", elem.Raw, "host", elem.SplunkServer, "firstLine", firstLine) + trimFirstLine := strings.TrimSuffix(firstLine, "\n") + if strings.Compare(elem.Raw, trimFirstLine) == 0 { + testcaseEnvInst.Log.Info("Found search results in _raw and splunk_server", "key", key, "podName", podName, "elem", elem) + found = true + } + } + Expect(found).To(Equal(true), "Incorrect search results %s", searchResults) +} diff --git a/test/licensemanager/manager_suite_test.go b/test/licensemanager/licensemanager_suite_test.go similarity index 77% rename from test/licensemanager/manager_suite_test.go rename to test/licensemanager/licensemanager_suite_test.go index d504c3685..8a58bf6da 100644 --- a/test/licensemanager/manager_suite_test.go +++ b/test/licensemanager/licensemanager_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemanager import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lmanager-" + testenv.RandomDNSName(3) diff --git a/test/licensemanager/lm_appfw_test.go b/test/licensemanager/lm_appfw_test.go new file mode 100644 index 000000000..4089e1a54 --- /dev/null +++ b/test/licensemanager/lm_appfw_test.go @@ -0,0 +1,45 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemanager + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Licensemanager App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var config *LicenseTestConfig + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + + config = NewLicenseManagerConfig() + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) + }) + }) +}) diff --git a/test/licensemanager/lm_s1_test.go b/test/licensemanager/lm_test.go similarity index 53% rename from test/licensemanager/lm_s1_test.go rename to test/licensemanager/lm_test.go index 035ab730f..634d880e5 100644 --- a/test/licensemanager/lm_s1_test.go +++ b/test/licensemanager/lm_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,8 @@ package licensemanager import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" ) @@ -31,35 +28,13 @@ var _ = Describe("Licensemanager test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1) with License Manager", func() { @@ -67,4 +42,16 @@ var _ = Describe("Licensemanager test", func() { RunLMS1Test(ctx, deployment, testcaseEnvInst, config) }) }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { + RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) + }) + + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster) with License Manager", func() { + It("licensemanager, integration, m4: Splunk Operator can configure License Manager with Indexers and Search Heads in M4 SVA", func() { + RunLMM4Test(ctx, deployment, testcaseEnvInst, config) + }) + }) }) diff --git a/test/licensemanager/lm_test_shared.go b/test/licensemanager/lm_test_shared.go index 205fd2c74..0da5f6916 100644 --- a/test/licensemanager/lm_test_shared.go +++ b/test/licensemanager/lm_test_shared.go @@ -20,108 +20,94 @@ import ( "path/filepath" "time" + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" "github.com/splunk/splunk-operator/test/testenv" - - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" ) +// LicenseTestConfig holds the version-specific (V3/V4) deployment and verification +// callbacks used by the license manager test functions. type LicenseTestConfig struct { - DeployStandaloneWithLM func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) - LicenseManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - ClusterManagerReady func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) - DeployMultisiteClusterWithSearchHead func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas, siteCount int, mcRef string) error - DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) - LicenseManagerPodName string - LicenseManagerSpecType string + *testenv.ClusterReadinessConfig + DeployLicenseManagerWithGivenSpec func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) + BuildLMAppFrameworkSpec func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} + LicenseManagerPodName string +} + +// newLicenseCommonSplunkSpec returns a CommonSplunkSpec pre-configured with the +// license config map volume and license URL. +func newLicenseCommonSplunkSpec(testcaseEnvInst *testenv.TestCaseEnv) enterpriseApi.CommonSplunkSpec { + return enterpriseApi.CommonSplunkSpec{ + Volumes: []corev1.Volume{{ + Name: "licenses", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: testcaseEnvInst.GetLMConfigMap(), + }, + }, + }, + }}, + LicenseURL: "/mnt/licenses/enterprise.lic", + Spec: enterpriseApi.Spec{ImagePullPolicy: "Always", Image: testcaseEnvInst.GetSplunkImage()}, + } } +// NewLicenseMasterConfig returns a LicenseTestConfig for V3 (LicenseMaster) tests. func NewLicenseMasterConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterMasterReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV3(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseMasterWithGivenSpec(ctx, name, spec.(enterpriseApiV3.LicenseMasterSpec)) }, - LicenseManagerPodName: testenv.LicenseMasterPod, - LicenseManagerSpecType: "v3", + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApiV3.LicenseMasterSpec{ + CommonSplunkSpec: newLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseMasterPod, } } +// NewLicenseManagerConfig returns a LicenseTestConfig for V4 (LicenseManager) tests. func NewLicenseManagerConfig() *LicenseTestConfig { return &LicenseTestConfig{ - DeployStandaloneWithLM: func(ctx context.Context, deployment *testenv.Deployment, name string, mcRef string) (*enterpriseApi.Standalone, error) { - return deployment.DeployStandaloneWithLM(ctx, name, mcRef) - }, - LicenseManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) - }, - ClusterManagerReady: func(ctx context.Context, deployment *testenv.Deployment, testcaseEnv *testenv.TestCaseEnv) { - testcaseEnv.VerifyClusterManagerReady(ctx, deployment) - }, - DeployMultisiteClusterWithSearchHead: func(ctx context.Context, deployment *testenv.Deployment, name string, indexerReplicas int, siteCount int, mcRef string) error { - return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) - }, + ClusterReadinessConfig: testenv.NewClusterReadinessConfigV4(), DeployLicenseManagerWithGivenSpec: func(ctx context.Context, deployment *testenv.Deployment, name string, spec interface{}) (interface{}, error) { return deployment.DeployLicenseManagerWithGivenSpec(ctx, name, spec.(enterpriseApi.LicenseManagerSpec)) }, - LicenseManagerPodName: testenv.LicenseManagerPod, - LicenseManagerSpecType: "v4", + BuildLMAppFrameworkSpec: func(testcaseEnvInst *testenv.TestCaseEnv, appFrameworkSpec enterpriseApi.AppFrameworkSpec) interface{} { + return enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: newLicenseCommonSplunkSpec(testcaseEnvInst), + AppFrameworkConfig: appFrameworkSpec, + } + }, + LicenseManagerPodName: testenv.LicenseManagerPod, } } -func downloadLicenseAndCreateConfigMap(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv) { - downloadDir := "licenseFolder" - +// downloadAppFiles downloads app files from the appropriate cloud provider. +func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { var err error - var licenseFilePath string - - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err = testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - case "azure": - licenseFilePath, err = testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - case "gcp": - licenseFilePath, err = testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) -} - -func downloadAppFiles(ctx context.Context, testDataS3Bucket, azureDataContainer, appDir, downloadDir string, appFileList []string, version string) { switch testenv.ClusterProvider { case "eks": - err := testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromS3(testDataS3Bucket, appDir, downloadDir, appFileList) case "azure": containerName := "/" + azureDataContainer + "/" + appDir - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDir, containerName, appFileList) case "gcp": - err := testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) + err = testenv.DownloadFilesFromGCP(testDataS3Bucket, appDir, downloadDir, appFileList) } + + Expect(err).To(Succeed(), fmt.Sprintf("Unable to download %s app files", version)) } +// uploadAppFiles uploads app files to the appropriate cloud provider and returns the uploaded file paths. func uploadAppFiles(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, testS3Bucket, testDir, downloadDir string, appFileList []string, version string) []string { var uploadedFiles []string var err error @@ -144,6 +130,7 @@ func uploadAppFiles(ctx context.Context, testcaseEnvInst *testenv.TestCaseEnv, t return uploadedFiles } +// deleteUploadedFiles removes previously uploaded app files from the appropriate cloud provider. func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps []string) { switch testenv.ClusterProvider { case "eks": @@ -156,102 +143,81 @@ func deleteUploadedFiles(ctx context.Context, testS3Bucket string, uploadedApps } } +// verifyLMAppsOnPod verifies that apps are copied and installed on the License Manager pod. +// The updated flag controls whether apps are expected to be updated versions. +func verifyLMAppsOnPod(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, podName []string, appList []string, updated bool) { + testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appList, true, enterpriseApi.ScopeLocal) + testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appList, updated, "enabled", updated, false) +} + +// verifyLMConfiguredOnCluster verifies that the License Manager is configured on +// the given indexer pods, on search head pods, and on the Monitoring Console. +func verifyLMConfiguredOnCluster(ctx context.Context, deployment *testenv.Deployment, indexerPods []string) { + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), 3, false, 0) + testenv.VerifyLMConfiguredOnPods(ctx, deployment, append(indexerPods, shPods...)) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) +} + +// RunLMS1Test deploys a Standalone with License Manager and Monitoring Console, +// then verifies LM is configured on the standalone and MC pods. func RunLMS1Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Create standalone Deployment with License Manager/Master + // Create Standalone deployment with License Manager/Master mcRef := deployment.GetName() standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with LM") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) // Wait for Standalone to be in READY status testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + // Deploy and verify Monitoring Console + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify livenessProbe and readinessProbe config object and scripts + testcaseEnvInst.VerifyProbeConfigAndScripts(ctx, deployment, false) - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testcaseEnvInst.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify License Manager/Master is configured on standalone instance + // Verify License Manager/Master is configured on Standalone instance standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testenv.VerifyLMConfiguredOnPod(ctx, deployment, standalonePodName) // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + testenv.VerifyLMConfiguredOnMC(ctx, deployment) } +// RunLMC3Test deploys a C3 cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. func RunLMC3Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Deploy Single site Cluster with License Manager/Master + // Deploy single site Cluster with License Manager/Master mcRef := deployment.GetName() err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + config.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst) - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), 3, false, 0) + verifyLMConfiguredOnCluster(ctx, deployment, indexerPods) } +// RunLMC3AppFrameworkTest deploys a License Manager with App Framework, verifies V1 apps +// are installed, upgrades to V2 apps, and verifies the updated apps. func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, testenvInstance *testenv.TestEnv, config *LicenseTestConfig) { var ( appListV1 []string appListV2 []string testS3Bucket = os.Getenv("TEST_INDEXES_S3_BUCKET") testDataS3Bucket = os.Getenv("TEST_BUCKET") - AzureDataContainer = os.Getenv("TEST_CONTAINER") + azureDataContainer = os.Getenv("TEST_CONTAINER") appDirV1 = testenv.AppLocationV1 appDirV2 = testenv.AppLocationV2 currDir, _ = os.Getwd() @@ -267,17 +233,17 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appFileList := testenv.GetAppFileList(appListV1) // Download V1 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion) + downloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV1, downloadDirV1, appFileList, appVersion) // Upload V1 apps testDir = "lm-" + testenv.RandomDNSName(4) uploadedFiles := uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV1, appFileList, appVersion) uploadedApps = append(uploadedApps, uploadedFiles...) - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) - // Create App framework Spec + // Create app framework spec volumeName := "lm-test-volume-" + testenv.RandomDNSName(3) var volumeSpec []enterpriseApi.VolumeSpec switch testenv.ClusterProvider { @@ -289,17 +255,17 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment volumeSpec = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "gcp", "blob", testenv.GetDefaultS3Region())} } - // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment + // AppSourceDefaultSpec: Remote Storage volume name and scope of app deployment appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ VolName: volumeName, Scope: enterpriseApi.ScopeLocal, } - // appSourceSpec: App source name, location and volume name and scope from appSourceDefaultSpec + // appSourceSpec: app source name, location and volume name and scope from appSourceDefaultSpec appSourceName := "lm-" + testenv.RandomDNSName(3) appSourceSpec := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceName, testDir, appSourceDefaultSpec)} - // appFrameworkSpec: AppSource settings, Poll Interval, volumes, appSources on volumes + // appFrameworkSpec: AppSource settings, poll interval, volumes, appSources on volumes appFrameworkSpec := enterpriseApi.AppFrameworkSpec{ Defaults: appSourceDefaultSpec, AppsRepoPollInterval: 60, @@ -307,69 +273,18 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment AppSources: appSourceSpec, } - var spec interface{} - if config.LicenseManagerSpecType == "v3" { - spec = enterpriseApiV3.LicenseMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } else { - spec = enterpriseApi.LicenseManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Volumes: []corev1.Volume{ - { - Name: "licenses", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testcaseEnvInst.GetLMConfigMap(), - }, - }, - }, - }, - }, - LicenseURL: "/mnt/licenses/enterprise.lic", - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - } + spec := config.BuildLMAppFrameworkSpec(testcaseEnvInst, appFrameworkSpec) // Deploy the License Manager/Master with App Framework - var err error - _, err = config.DeployLicenseManagerWithGivenSpec(ctx, deployment, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy LM with App framework") + _, err := config.DeployLicenseManagerWithGivenSpec(ctx, deployment, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy LM with App Framework") // Wait for License Manager/Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) + // Verify apps are copied and installed on License Manager/Master podName := []string{fmt.Sprintf(config.LicenseManagerPodName, deployment.GetName(), 0)} - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV1, true, enterpriseApi.ScopeLocal) - - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV1, false, "enabled", false, false) + verifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV1, false) // Delete files uploaded deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) @@ -381,7 +296,7 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment appVersion = "V2" // Download V2 Apps - downloadAppFiles(ctx, testDataS3Bucket, AzureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion) + downloadAppFiles(ctx, testDataS3Bucket, azureDataContainer, appDirV2, downloadDirV2, appFileList, appVersion) // Upload V2 apps uploadedFiles = uploadAppFiles(ctx, testcaseEnvInst, testS3Bucket, testDir, downloadDirV2, appFileList, appVersion) @@ -391,13 +306,10 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment Expect(err).To(Succeed(), "Timed out waiting for LicenseManager to reach Ready phase") // Verify LM stays in ready state - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Verify apps are copied at the correct location on License Manager/Master (/etc/apps/) - testcaseEnvInst.VerifyAppsCopied(ctx, deployment, testenvInstance.GetName(), podName, appListV2, true, enterpriseApi.ScopeLocal) + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - // Verify apps are installed on License Manager/Master - testcaseEnvInst.VerifyAppInstalled(ctx, deployment, testcaseEnvInst.GetName(), podName, appListV2, true, "enabled", true, false) + // Verify apps are copied and installed on License Manager/Master + verifyLMAppsOnPod(ctx, deployment, testcaseEnvInst, testenvInstance, podName, appListV2, true) // Delete files uploaded deleteUploadedFiles(ctx, testS3Bucket, uploadedApps) @@ -407,55 +319,30 @@ func RunLMC3AppFrameworkTest(ctx context.Context, deployment *testenv.Deployment os.RemoveAll(downloadDirV2) } +// RunLMM4Test deploys a multisite cluster with License Manager and Monitoring Console, +// then verifies LM is configured on indexers, search heads, and MC. func RunLMM4Test(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *LicenseTestConfig) { - // Download License File - downloadLicenseAndCreateConfigMap(ctx, testcaseEnvInst) + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) // Deploy Multisite Cluster with License Manager/Master and Search Head siteCount := 3 mcRef := deployment.GetName() - err := config.DeployMultisiteClusterWithSearchHead(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) + err := config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcRef) Expect(err).To(Succeed(), "Unable to deploy cluster") - // Wait for Cluster Manager to be in READY status - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) - // Wait for Indexers to be in READY status - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + deployMCAndVerifyRFSF(ctx, deployment, testcaseEnvInst, mcRef) - // Verify Multisite Indexer Cluster status - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Wait for Search Head Cluster to be in READY status - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Deploy Monitoring Console - mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Verify License Manager/Master is configured on indexers, search heads, and MC + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) + verifyLMConfiguredOnCluster(ctx, deployment, indexerPods) +} - // Verify RF SF is met +// deployMCAndVerifyRFSF deploys a Monitoring Console and verifies RF/SF is met. +func deployMCAndVerifyRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, mcRef string) { + _ = testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcRef, deployment.GetName()) testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify License Manager/Master is configured on indexers - indexerPodName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 2, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - indexerPodName = fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 3, 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, indexerPodName) - - // Verify License Manager/Master is configured on SHs - searchHeadPodName := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 1) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - searchHeadPodName = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 2) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, searchHeadPodName) - - // Verify License Manager/Master is configured on Monitoring Console - monitoringConsolePodName := fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName()) - testenv.VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) } diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go deleted file mode 100644 index 8e44449c9..000000000 --- a/test/licensemanager/manager_lm_c3_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure License Manager with Indexers and Search Heads in C3 SVA", func() { - RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Manager", func() { - It("licensemanager, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemanager/manager_lm_m4_test.go b/test/licensemanager/manager_lm_m4_test.go deleted file mode 100644 index aedcd4dee..000000000 --- a/test/licensemanager/manager_lm_m4_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemanager - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemanager test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = NewLicenseManagerConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Manager", func() { - It("licensemanager, integration, m4: Splunk Operator can configure License Manager with indexers and search head in M4 SVA", func() { - RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_appfw_test.go b/test/licensemaster/lm_appfw_test.go new file mode 100644 index 000000000..0536c05aa --- /dev/null +++ b/test/licensemaster/lm_appfw_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package licensemaster + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/splunk/splunk-operator/test/licensemanager" + "github.com/splunk/splunk-operator/test/testenv" +) + +var _ = Describe("Licensemaster App Framework test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + var config *licensemanager.LicenseTestConfig + ctx := context.TODO() + + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") + + config = licensemanager.NewLicenseMasterConfig() + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { + licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) + }) + }) +}) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go deleted file mode 100644 index ed2f83f7f..000000000 --- a/test/licensemaster/lm_c3_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { - licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster) with License Master", func() { - It("licensemaster, integration, c3: Splunk Operator can configure a C3 SVA and have apps installed locally on LM", func() { - licensemanager.RunLMC3AppFrameworkTest(ctx, deployment, testcaseEnvInst, testenvInstance, config) - }) - }) -}) diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_m4_test.go deleted file mode 100644 index 1e4c6fb65..000000000 --- a/test/licensemaster/lm_m4_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package licensemaster - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/licensemanager" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Licensemaster test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var config *licensemanager.LicenseTestConfig - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster) with License Master", func() { - It("licensemaster, integration, m4: Splunk Operator can configure License Master with indexers and search head in M4 SVA", func() { - licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) - }) - }) - -}) diff --git a/test/licensemaster/lm_suite_test.go b/test/licensemaster/lm_suite_test.go index 9df3f4576..ceba436ed 100644 --- a/test/licensemaster/lm_suite_test.go +++ b/test/licensemaster/lm_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package licensemaster import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "lm-" + testenv.RandomDNSName(3) diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_test.go similarity index 55% rename from test/licensemaster/lm_s1_test.go rename to test/licensemaster/lm_test.go index cc5c92e6b..55c5010af 100644 --- a/test/licensemaster/lm_s1_test.go +++ b/test/licensemaster/lm_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,8 @@ package licensemaster import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/licensemanager" "github.com/splunk/splunk-operator/test/testenv" @@ -33,33 +30,13 @@ var _ = Describe("Licensemaster test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") config = licensemanager.NewLicenseMasterConfig() - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1) with License Master", func() { @@ -67,4 +44,16 @@ var _ = Describe("Licensemaster test", func() { licensemanager.RunLMS1Test(ctx, deployment, testcaseEnvInst, config) }) }) + + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, c3: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { + licensemanager.RunLMC3Test(ctx, deployment, testcaseEnvInst, config) + }) + }) + + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster) with License Master", func() { + It("licensemaster, integration, m4: Splunk Operator can configure License Master with Indexers and Search Heads in M4 SVA", func() { + licensemanager.RunLMM4Test(ctx, deployment, testcaseEnvInst, config) + }) + }) }) diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go deleted file mode 100644 index 1cc3b89f3..000000000 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ /dev/null @@ -1,1101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package monitoringconsoletest - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -var _ = Describe("Monitoring Console test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Deploy Monitoring Console", func() { - It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { - /* - Test Steps - 1. Deploy Monitoring Console - 2. Deploy Standalone - 3. Wait for Monitoring Console status to go back to READY - 4. Verify Standalone configured in Monitoring Console Config Map - 5. Verify Monitoring Console Pod has correct peers in Peer List - --------------- RECONFIG WITH NEW MC -------------------------- - 6. Reconfig S1 with 2nd Monitoring Console Name - 7. Check 2nd Monitoring Console Config Map to verify s1 - 8. Deploy 2nd Monitoring Console Pod - 9. Verify Standalone pod is configured on Monitoring Console Pod - 10. Verify 1st Monitoring Console Config Map is not configured with S1 - 11. Verify 1st Monitoring Console Pod is not configured with S1 - */ - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Create Standalone Spec and apply - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### - - // Reconfig S1 with 2nd Monitoring Console Name - mcTwoName := deployment.GetName() + "-two" - err = deployment.GetInstance(ctx, standaloneOneName, standaloneOne) - Expect(err).To(Succeed(), "Unable to get instance of Standalone") - standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - - // Update Standalone with 2nd MC - err = deployment.UpdateCR(ctx, standaloneOne) - Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - - // Deploy 2nd MC Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Second Monitoring Console Pod") - - // Verify 2nd Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on SECOND MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcTwoName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in SECOND MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcTwoName, true, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is not configured in MC ONE Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod NOT ON FIRST MC Config Map after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod Not in MC ONE Peer List - testcaseEnvInst.Log.Info("Check standalone NOT ON FIRST MC Peer list after Standalone RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - // CSPL-901 standaloneTwoName := deployment.GetName() + "-two" - standaloneTwoName := "standalone-" + testenv.RandomDNSName(3) - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete Standlone TWO of the standalone and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - - Context("Standalone deployment with Scale up", func() { - It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { - /* - Test Steps - 1. Deploy Standalone - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Scale Standalone to 2 REPLICAS - 8. Wait for Second Standalone POD to come up and PHASE to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone PODS configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneName := deployment.GetName() - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standalone to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 1, false, 0) - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Scale Standalone instance - testcaseEnvInst.Log.Info("Scaling Standalone CR") - scaledReplicaCount := 2 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale Standalone") - - // Ensure standalone is scaling up - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to Updating Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - - // Check Standalone is configure in MC Config Map - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings - 6. Verify Monitoring Console Pod has peers(indexers) in Peer string - 7. Scale SH Cluster - 8. Scale Indexer Count - 9. Add a standalone - 10. Verify Standalone is configured in MC Config Map and Peer String - 11. Verify SH are configured in MC Config Map and Peers String - 12. Verify Indexers are configured in Peer String - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Cluster Manager to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // Scale Search Head Cluster - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) - - // Scale indexers - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready Phase - // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) - - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managermc1, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { - /* - Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod - */ - - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Wait for Monitoring console Pod to be configured with all search head - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ################# Update Monitoring Console In Cluster Manager CR ################################## - - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Manager Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") - - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify MC is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo, testcaseEnvInst) - - // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### - - // Check Cluster Manager in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) - - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") - - // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### - - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) - - // Check Cluster Manager Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) - - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - }) - }) - - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("managermc2, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { - /* - Test Steps - 1. Deploy Multisite Indexer Cluster - 2. Deploy Monitoring Console - 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Manager to use 2nd Monitoring Console - 9. Verify Cluster Manager is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Manager not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod - */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) - - // Ensure Cluster Manager Goes to Updating Phase - //testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // Check Cluster Manager in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcTwoName, true) - - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Manager NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Manager RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { - /* - Test Steps - 1. Deploy Standalone with name "search-head-adhoc" - 2. Wait for Standalone to go to READY - 3. Deploy Monitoring Console - 4. Wait for Monitoring Console status to be READY - 5. Verify Standalone configured in Monitoring Console Config Map - 6. Verify Monitoring Console Pod has correct peers in Peer List - 7. Deploy 2nd Standalone with name "search-head" - 8. Wait for Second Standalone to be READY - 9. Wait for Monitoring Console status to go UPDATING then READY - 10. Verify both Standalone configured in Monitoring Console Config Map - 11. Verify both Standalone configured in Monitoring Console Pod Peers String - 12. Delete 2nd Standalone - 13. Wait for Monitoring Console to go to UPDATING then READY - 14. Verify only first Standalone configured in Monitoring Console Config Map - 15. Verify only first Standalone configured in Monitoring Console Pod Peers String - */ - - standaloneOneName := "search-head-adhoc" - mcName := deployment.GetName() - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneOne, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneOneName, spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") - - // Wait for standaloneOne to be in READY Status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneOneName, standaloneOne) - - // Deploy MC and wait for MC to be READY - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneOne is configured in MC Config Map - standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneOne Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Add another standalone instance in namespace - testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") - standaloneTwoName := "search-head" - // Configure Resources on second standalone CSPL-555 - standaloneTwoSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("4Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("0.2"), - "memory": resource.MustParse("256Mi"), - }, - }, - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for standalone two to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) - - // wait for custom resource(CR) resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Vrify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check standaloneTwo is configure in MC Config Map - standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) - - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check standaloneTwo Pod in MC Peer List - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list after adding new standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Delete standaloneTwo and ensure MC is updated - testcaseEnvInst.Log.Info("Deleting second standalone deployment to namespace", "Standalone Name", standaloneTwoName) - deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) - err = deployment.DeleteCR(ctx, standaloneTwo) - Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone is configure in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, true) - - // Check Standalone Pod in MC Peer List - testcaseEnvInst.Log.Info("Check Standalone One Pod in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, true, false) - - // Check Standalone TWO NOT configured in MC Config Map - standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) - - testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, standalonePods, "SPLUNK_STANDALONE_URL", mcName, false) - - // Check Standalone Pod TWO NOT configured MC Peer List - testcaseEnvInst.Log.Info("Check Standalone Two Pod NOT in MC Peer list after deleting second standalone") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, standalonePods, mcName, false, false) - - }) - }) - -}) diff --git a/test/monitoring_console/monitoring_console_suite_test.go b/test/monitoring_console/monitoring_console_suite_test.go index 83bf2060d..167041418 100644 --- a/test/monitoring_console/monitoring_console_suite_test.go +++ b/test/monitoring_console/monitoring_console_suite_test.go @@ -1,23 +1,20 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package monitoringconsoletest import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -25,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "mc-" + testenv.RandomDNSName(3) @@ -41,13 +29,9 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) - sc, _ := GinkgoConfiguration() - sc.Timeout = 240 * time.Minute - - RunSpecs(t, "Running "+testSuiteName, sc) + RunSpecs(t, "Running "+testSuiteName) } var _ = BeforeSuite(func() { diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index 87605c825..702927551 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,49 +19,69 @@ import ( "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) -var _ = Describe("Monitoring Console test", func() { +// masterManagerMCConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 reconfig and M4 MC reconfig test tables. +var masterManagerMCConfigs = []MCVersionConfig{ + { + MCReconfigParams: MCReconfigParams{CMServiceNameFmt: testenv.ClusterMasterServiceName, CMURLKey: "SPLUNK_CLUSTER_MASTER_URL"}, + NamePrefix: "master", + Label: "mastermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() interface{} { return &enterpriseApiV3.ClusterMaster{} }, + VerifyCMReady: func(ctx context.Context, te *testenv.TestCaseEnv, d *testenv.Deployment) { + te.VerifyClusterMasterReady(ctx, d) + }, + SHCReconfigTimeout: 0, + VerifyMCTwoReadyAfterSHC: true, + }, + { + MCReconfigParams: MCReconfigParams{CMServiceNameFmt: testenv.ClusterManagerServiceName, CMURLKey: splcommon.ClusterManagerURL}, + NamePrefix: "", + Label: "managermc", + DeployC3WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error { + return d.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, name, replicas, shc, mcRef) + }, + DeployM4WithMC: func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error { + return d.DeployMultisiteClusterWithMonitoringConsole(ctx, name, replicas, siteCount, mcRef, shc) + }, + NewCMObject: func() interface{} { return &enterpriseApi.ClusterManager{} }, + VerifyCMReady: func(ctx context.Context, te *testenv.TestCaseEnv, d *testenv.Deployment) { + te.VerifyClusterManagerReady(ctx, d) + }, + SHCReconfigTimeout: 5 * time.Minute, + VerifyMCTwoReadyAfterSHC: false, + }, +} + +// Master (V3) Monitoring Console tests +var _ = Describe("Monitoring Console test (master)", func() { var testcaseEnvInst *testenv.TestCaseEnv var deployment *testenv.Deployment ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("mastermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { /* Test Steps @@ -69,7 +89,7 @@ var _ = Describe("Monitoring Console test", func() { 2. Deploy Monitoring Console 3. Wait for Monitoring Console status to go back to READY 4. Verify SH are configured in MC Config Map - 5. VerifyMonitoring Console Pod has Search Heads in Peer strings + 5. Verify Monitoring Console Pod has Search Heads in Peer strings 6. Verify Monitoring Console Pod has peers(indexers) in Peer string 7. Scale SH Cluster 8. Scale Indexer Count @@ -83,504 +103,345 @@ var _ = Describe("Monitoring Console test", func() { defaultIndexerReplicas := 3 mcName := deployment.GetName() - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Deploy and verify Monitoring Console + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") + // Deploy and verify C3 cluster with MC + testcaseEnvInst.DeployAndVerifyC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, mcName) - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) // Wait for Cluster Master to appear in Monitoring Console Config Map - err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) + err := testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true, 2*time.Minute) Expect(err).To(Succeed(), "Timed out waiting for Cluster Master in MC ConfigMap") - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + // Verify MC configuration for C3 cluster + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, defaultSHReplicas, defaultIndexerReplicas, true) // Scale Search Head Cluster scaledSHReplicas := defaultSHReplicas + 1 testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - shcName := deployment.GetName() + "-shc" - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update Replicas of SHC - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") - - // Ensure Search Head cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) // Scale indexers scaledIndexerReplicas := defaultIndexerReplicas + 1 testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - idxcName := deployment.GetName() + "-idxc" - - // Get instance of current Indexer CR with latest config - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") - - // Ensure Indxer cluster scales up and go to ScalingUp phase - testcaseEnvInst.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) // get revision number of the resource resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Deploy Standalone Pod - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance") + // Deploy Standalone with MC reference + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Ensure Indexer cluster go to Ready phase + // Ensure Indexer Cluster goes to Ready phase testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - // Ensure Search Head Cluster go to Ready Phase + // Ensure Search Head Cluster goes to Ready Phase // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + // wait for custom resource resource version to change and verify MC is ready + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - // Wait for MC to go to PENDING Phase - //testcaseEnvInst.VerifyMonitoringConsolePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhasePending) + // Verify Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), mcName, true) - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Standalone configured on Monitoring Console - testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, "SPLUNK_STANDALONE_URL", mcName, true) - - testcaseEnvInst.Log.Info("Check standalone instance in MC Peer list") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, mcName, true, false) + // Verify MC configuration after scale up + testcaseEnvInst.Log.Info("Verify MC configuration after Scale Up") + testcaseEnvInst.VerifyMCConfigForC3Cluster(ctx, deployment, deployment.GetName(), mcName, scaledSHReplicas, scaledIndexerReplicas, true) + }) + }) +}) - // Verify all Search Head Members are configured on Monitoring Console - shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) +// Manager (V4) Monitoring Console tests +var _ = Describe("Monitoring Console test (manager)", func() { - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) - // Check Monitoring console is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") - indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - }) + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + Context("Deploy Monitoring Console", func() { + It("smoke, monitoringconsole: can deploy MC CR and can be configured standalone", func() { /* Test Steps - 1. Deploy Single Site Indexer Cluster - 2. Deploy Monitoring Console + 1. Deploy Monitoring Console + 2. Deploy Standalone 3. Wait for Monitoring Console status to go back to READY - 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Monitoring Console Pod has peers in Peer string on MC Pod - --------------- RECONFIG CLUSTER MANAGER WITH NEW MC -------------------------- - 13. Reconfigure CM with Second MC - 14. Verify CM in config map of Second MC - 15. Create Second MC Pod - 16. Verify Indexers in second MC Pod - 17. Verify SHC not in second MC - 18. Verify SHC still present in first MC d - 19. Veirfy CM and Indexers not in first MC Pod - ---------------- RECONFIG SHC WITH NEW MC - 20. Configure SHC with Second MC - 21. Verify CM, DEPLOYER, Search Heads in config map of seconds MC - 22. Verify SHC in second MC pod - 23. Verify Indexers in Second MC Pod - 24. Verify CM and Deployer not in first MC CONFIG MAP - 24. Verify SHC, Indexers not in first MC Pod + 4. Verify Standalone configured in Monitoring Console Config Map + 5. Verify Monitoring Console Pod has correct peers in Peer List + --------------- RECONFIG WITH NEW MC -------------------------- + 6. Reconfig S1 with 2nd Monitoring Console Name + 7. Check 2nd Monitoring Console Config Map to verify s1 + 8. Deploy 2nd Monitoring Console Pod + 9. Verify Standalone pod is configured on Monitoring Console Pod + 10. Verify 1st Monitoring Console Config Map is not configured with S1 + 11. Verify 1st Monitoring Console Pod is not configured with S1 */ - defaultSHReplicas := 3 - defaultIndexerReplicas := 3 - mcName := deployment.GetName() - - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - err = deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Deploy Monitoring Console CRD + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) - - // Check Deployer in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - - // Check Search Head Pods in Monitoring Console Config Map - shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + // Create Standalone Spec and apply + standaloneOneName := deployment.GetName() + mcName := deployment.GetName() + standaloneOne := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) - // Check Monitoring console Pod is configured with all search head - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + // wait for custom resource resource version to change and verify MC is ready + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, true) - // ################# Update Monitoring Console In Cluster Master CR ################################## + // ######################### RECONFIGURE STANDALONE WITH SECOND MC ####################################### + // Reconfig S1 with 2nd Monitoring Console Name mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) + testenv.GetInstanceWithExpect(ctx, deployment, standaloneOne, standaloneOneName, "Unable to get instance of Standalone") + standaloneOne.Spec.MonitoringConsoleRef.Name = mcTwoName - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + // Update Standalone with 2nd MC + err := deployment.UpdateCR(ctx, standaloneOne) + Expect(err).To(Succeed(), "Unable to update Standalone with new MC Name") - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) + // Deploy 2nd MC Pod + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + // Check Standalone is configured in MC Two + testcaseEnvInst.Log.Info("Checking for Standalone on SECOND MC after Standalone RECONFIG") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcTwoName, true) - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) - - // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### - - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager in Monitoring Console Two Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) - - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Verify Indexers in Monitoring Console Pod TWO Config String after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) - - // Check Deployer NOT in Monitoring Console TWO Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) - - // Check Monitoring Console TWO is NOT configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) - - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console TWO Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) - - // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### - - // Verify Monitoring Console One Ready and stays in ready state + // Verify Monitoring Console One is Ready and stays in ready state testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) - - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Verify Indexers NOT in Monitoring Console One Pod Config String after Cluster Master Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) - - // Check Monitoring Console One is still configured with Search Head in namespace - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console One Config Map after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Cluster Manager Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - - // ################# Update Monitoring Console In SHC CR ################################## - - // Get instance of current SHC CR with latest config - shc := &enterpriseApi.SearchHeadCluster{} - shcName := deployment.GetName() + "-shc" - err = deployment.GetInstance(ctx, shcName, shc) - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Update SHC to use 2nd Montioring Console - shc.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to get update Monitoring Console in Search Head Cluster CRD") + // Check Standalone is NOT configured in MC One + testcaseEnvInst.Log.Info("Checking for Standalone NOT ON FIRST MC after Standalone RECONFIG") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneOneName, mcName, false) - // Ensure Search Head Cluster go to Ready Phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + }) + }) - // Verify MC is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + Context("Standalone deployment (S1)", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC with new standalone deployment", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, deployment.GetName(), "standalone-"+testenv.RandomDNSName(3)) + }) + }) - // ############################ VERIFICATOIN FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + Context("Standalone deployment with Scale up", func() { + It("managermc1, integration: can deploy a MC with standalone instance and update MC when standalone is scaled up", func() { + /* + Test Steps + 1. Deploy Standalone + 2. Wait for Standalone to go to READY + 3. Deploy Monitoring Console + 4. Wait for Monitoring Console status to be READY + 5. Verify Standalone configured in Monitoring Console Config Map + 6. Verify Monitoring Console Pod has correct peers in Peer List + 7. Scale Standalone to 2 REPLICAS + 8. Wait for Second Standalone POD to come up and PHASE to be READY + 9. Wait for Monitoring Console status to go UPDATING then READY + 10. Verify both Standalone PODS configured in Monitoring Console Config Map + 11. Verify both Standalone configured in Monitoring Console Pod Peers String + */ - // Check Cluster Master in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) + standaloneName := deployment.GetName() + mcName := deployment.GetName() + standalone := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneName, mcName) - // Check Deployer in Monitoring Console Two Config Map - testcaseEnvInst.Log.Info("Verify Deployer on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") - // Verify all Search Head Members are configured on Monitoring Console Two - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Two Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) + // Check Standalone is configured in MC + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, standaloneName, mcName, true) - testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) + // get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Check Monitoring console Two is configured with all Indexer in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC TWO after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + // Scale Standalone instance + testcaseEnvInst.Log.Info("Scaling Standalone CR") + scaledReplicaCount := 2 + standalone = &enterpriseApi.Standalone{} + testenv.GetInstanceWithExpect(ctx, deployment, standalone, deployment.GetName(), "Failed to get instance of Standalone") - // ############################ VERIFICATOIN FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + standalone.Spec.Replicas = int32(scaledReplicaCount) - // Verify MC ONE is Ready and stays in ready state before running verfications - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + err := deployment.UpdateCR(ctx, standalone) + Expect(err).To(Succeed(), "Failed to scale Standalone") - // Check Cluster Master Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify Cluster Manager NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) + // Ensure standalone is scaling up + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseScalingUp) - // Check DEPLOYER Not in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Verify DEPLOYER NOT in Monitoring Console One Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - // Verify all Search Head Members are Not configured on Monitoring Console One - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Config Map after SHC Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) + // wait for custom resource resource version to change and verify MC is ready + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - testcaseEnvInst.Log.Info("Verify Search Head Pods NOT on Monitoring Console ONE Pod after Search Head Reconfig") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneName, 2, false, 0) - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pod NOT on MC One after SHC Reconfig") - // testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + // Check both Standalone pods are configured in MC after scale up + testcaseEnvInst.Log.Info("Checking for Standalone Pods on MC after scale up") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) }) }) - Context("Multisite Clustered deployment (M4 - 3 Site clustered indexer, search head cluster)", func() { - It("mastermc, integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + It("managermc, smoke: MC can configure SHC, indexer instances after scale up and standalone in a namespace", func() { /* Test Steps - 1. Deploy Multisite Indexer Cluster + 1. Deploy Single Site Indexer Cluster 2. Deploy Monitoring Console 3. Wait for Monitoring Console status to go back to READY 4. Verify SH are configured in MC Config Map - 5. Verify SH are configured in peer strings on MC Pod - 6. Verify Indexers are configured in MC Config Map - 7. Verify Monitoring Console Pod has peers in Peer string on MC Pod - ############ CLUSTER MANAGER MC RECONFIG ################################# - 8. Configure Cluster Master to use 2nd Monitoring Console - 9. Verify Cluster Master is configured Config Maps of Second MC - 10. Deploy 2nd MC pod - 11. Verify Indexers in 2nd MC Pod - 12. Verify SHC not in 2nd MC CM - 13. Verify SHC not in 2nd MC Pod - 14. Verify Cluster Master not 1st MC Config Map - 15. Verify Indexers not in 1st MC Pod + 5. Verify Monitoring Console Pod has Search Heads in Peer strings + 6. Verify Monitoring Console Pod has peers(indexers) in Peer string + 7. Scale SH Cluster + 8. Scale Indexer Count + 9. Add a standalone + 10. Verify Standalone is configured in MC Config Map and Peer String + 11. Verify SH are configured in MC Config Map and Peers String + 12. Verify Indexers are configured in Peer String */ + defaultSHReplicas := 3 - defaultIndexerReplicas := 1 - siteCount := 3 + defaultIndexerReplicas := 3 mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) - Expect(err).To(Succeed(), "Unable to deploy Indexer Cluster with SHC") - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") - // Ensure indexers go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) + err := deployment.DeploySingleSiteClusterWithGivenMonitoringConsole(ctx, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - // Ensure indexer clustered is configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst.VerifyClusterManagerReady) - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, true) + // Wait for Cluster Manager to appear in Monitoring Console Config Map + err = testcaseEnvInst.WaitForPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterManagerServiceName, deployment.GetName())}, splcommon.ClusterManagerURL, mcName, true, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for Cluster Manager in MC ConfigMap") // Check Deployer in Monitoring Console Config Map testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) - // Deploy Monitoring Console Pod - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Check Monitoring console is configured with all search head instances in namespace + // Check Search Head Pods in Monitoring Console Config Map shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) - - testcaseEnvInst.Log.Info("Checking for Search Head on MC Config Map") testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - testcaseEnvInst.Log.Info("Checking for Search Head on MC Pod") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + // Wait for Monitoring console Pod to be configured with all search head + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false, 5*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC config") // Check Monitoring console is configured with all Indexer in Name Space - indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, 3) - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC POD") + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) - // ############ CLUSTER MANAGER MC RECONFIG ################################# - mcTwoName := deployment.GetName() + "-two" - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, cm) - - cm.Spec.MonitoringConsoleRef.Name = mcTwoName - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to update mcRef in Cluster Manager") + // Scale Search Head Cluster + scaledSHReplicas := defaultSHReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testcaseEnvInst.ScaleSearchHeadCluster(ctx, deployment, deployment.GetName(), scaledSHReplicas) - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, cm, resourceVersion) + // Scale indexers + scaledIndexerReplicas := defaultIndexerReplicas + 1 + testcaseEnvInst.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testcaseEnvInst.ScaleIndexerCluster(ctx, deployment, deployment.GetName(), scaledIndexerReplicas) - // Ensure Cluster Master Goes to Updating Phase - //testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) + // Deploy Standalone Pod + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) - // Deploy Monitoring Console Pod - mcTwo, err := deployment.DeployMonitoringConsole(ctx, mcTwoName, "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console Two instance") + // Ensure Indexer Cluster goes to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - // Verify Monitoring Console TWO is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + // Ensure Search Head Cluster goes to Ready Phase + // Adding this check in the end as SHC take the longest time to scale up due recycle of SHC members + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - // Check Cluster Master in Monitoring Console Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager on MC TWO CONFIG MAP after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcTwoName, true) + // wait for custom resource resource version to change and verify MC is ready + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) - // Check Monitoring Console TWO is configured with all Indexers in Name Space - testcaseEnvInst.Log.Info("Checking for Indexer Pods on MC TWO POD after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + // Check Standalone configured on Monitoring Console + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC") + testcaseEnvInst.VerifyStandaloneInMC(ctx, deployment, deployment.GetName(), mcName, true) - // Check Monitoring console Two is NOT configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) + // Verify all Search Head Members are configured on Monitoring Console + shPods = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas, false, 0) - testcaseEnvInst.Log.Info("Checking for Search Head NOT CONFIGURED on MC TWO Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) + testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Config Map after Scale Up") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) - // Verify Monitoring Console One is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + testcaseEnvInst.Log.Info("Verify Search Head Pods on Monitoring Console Pod after Scale Up") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) - // Check Cluster Master NOT configured on Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Cluster Manager NOT in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.ClusterMasterServiceName, deployment.GetName())}, "SPLUNK_CLUSTER_MASTER_URL", mcName, false) + // Check Monitoring console is configured with all Indexer in Name Space + testcaseEnvInst.Log.Info("Checking for Indexer Pod on MC after Scale Up") + indexerPods = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas, false, 0) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) + }) + }) - // Check Monitoring console One is Not configured with all Indexer in Name Space - // CSPL-619 - // testcaseEnvInst.Log.Info("Checking for Indexer Pods Not on MC one POD after Cluster Master RECONFIG") - //testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, false, true) + Context("Standalone deployment (S1)", func() { + It("managermc2, integration: can deploy a MC with standalone instance and update MC with new standalone deployment of similar names", func() { + RunS1StandaloneAddDeleteMCTest(ctx, deployment, testcaseEnvInst, "search-head-adhoc", "search-head") + }) + }) - // Check Deployer in Monitoring Console One Config Map - testcaseEnvInst.Log.Info("Checking for Deployer in MC One Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) +}) - // Check Monitoring console One is configured with all search head instances in namespace - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Config Map after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) +// C3 reconfig and M4 tests — V3 (master) and V4 (manager) variants +var _ = Describe("Monitoring Console reconfig tests", func() { - testcaseEnvInst.Log.Info("Checking for Search Head on MC ONE Pod after Cluster Manager RECONFIG") - testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + // C3 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Clustered deployment C3 reconfig ("+cfg.Label+")", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure to new MC", func() { + RunC3MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) }) - }) + } + + // M4 reconfig tests + for _, cfg := range masterManagerMCConfigs { + cfg := cfg + Context("Multisite Clustered deployment M4 reconfig ("+cfg.Label+")", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, cfg.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(cfg.Label+", integration: MC can configure SHC, indexer instances and reconfigure Cluster Manager to new Monitoring Console", func() { + RunM4MCReconfigTest(ctx, deployment, testcaseEnvInst, cfg) + }) + }) + } }) diff --git a/test/monitoring_console/monitoring_console_test_shared.go b/test/monitoring_console/monitoring_console_test_shared.go new file mode 100644 index 000000000..ce911a3e7 --- /dev/null +++ b/test/monitoring_console/monitoring_console_test_shared.go @@ -0,0 +1,395 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package monitoringconsoletest + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// MCVersionConfig captures the API-version-specific behaviour that differs +// between V3 (master) and V4 (manager) monitoring console tests. +type MCVersionConfig struct { + MCReconfigParams + + NamePrefix string + Label string + + // DeployC3WithMC deploys a C3 single-site cluster with the given MC ref. + DeployC3WithMC func(ctx context.Context, d *testenv.Deployment, name string, replicas int, shc bool, mcRef string) error + + // DeployM4WithMC deploys an M4 multisite cluster with the given MC ref. + DeployM4WithMC func(ctx context.Context, d *testenv.Deployment, name string, replicas int, siteCount int, mcRef string, shc bool) error + + // NewCMObject returns a new, empty cluster-coordinator CR + // (*ClusterMaster for V3, *ClusterManager for V4). + NewCMObject func() interface{} + + // VerifyCMReady asserts the cluster coordinator has reached Ready phase. + VerifyCMReady func(ctx context.Context, te *testenv.TestCaseEnv, d *testenv.Deployment) + + // SHCReconfigTimeout is the timeout used when verifying MC config strings + // after an SHC MC-ref reconfig (0 means use the synchronous check). + SHCReconfigTimeout time.Duration + + // VerifyMCTwoReadyAfterSHC controls whether MC Two is explicitly + // verified ready after the SHC reconfig step. + VerifyMCTwoReadyAfterSHC bool +} + +// verifyMCConfigForCluster verifies that the CM, deployer, search heads, and indexers +// are all correctly registered in the MC config map and pod config string. +// It uses the service name and URL key from the MCVersionConfig so it works for both V3 and V4. +func verifyMCConfigForCluster(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + cfg MCVersionConfig, mcName string, shPods, indexerPods []string) { + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(cfg.CMServiceNameFmt, deployment.GetName())}, cfg.CMURLKey, mcName, true) + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, true, true) +} + +// RunM4MCReconfigTest deploys an M4 multisite cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager to point +// to a second MC and verifies both MCs are updated correctly. +func RunM4MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 1 + siteCount := 3 + mcName := deployment.GetName() + + err := cfg.DeployM4WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, siteCount, mcName, true) + Expect(err).To(Succeed(), "Unable to deploy multisite cluster") + + // Ensure cluster coordinator and all M4 components are ready + cfg.VerifyCMReady(ctx, testcaseEnvInst, deployment) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Generate pod name slices for verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), defaultIndexerReplicas, true, siteCount) + + // Verify MC configuration for M4 cluster + verifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods) + + // ############ CLUSTER MANAGER MC RECONFIG ################################# + mcTwoName := deployment.GetName() + "-two" + cm := cfg.NewCMObject() + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, cm, deployment.GetName(), mcTwoName) + + cfg.VerifyCMReady(ctx, testcaseEnvInst, deployment) + + // Deploy and verify Monitoring Console Two + testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, false) + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, true) +} + +// RunC3MCReconfigTest deploys a C3 single-site cluster with a Monitoring Console, +// verifies the MC configuration, then reconfigures the Cluster Manager and SHC +// to point to a second MC and verifies both MCs are updated correctly. +func RunC3MCReconfigTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, cfg MCVersionConfig) { + defaultSHReplicas := 3 + defaultIndexerReplicas := 3 + mcName := deployment.GetName() + + // Deploy Monitoring Console Pod + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), "") + + err := cfg.DeployC3WithMC(ctx, deployment, deployment.GetName(), defaultIndexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + // Ensure C3 cluster is ready + testcaseEnvInst.VerifyC3ClusterReady(ctx, deployment, func(ctx2 context.Context, d *testenv.Deployment) { + cfg.VerifyCMReady(ctx2, testcaseEnvInst, d) + }) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Generate pod name slices for verification + shPods := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), defaultSHReplicas, false, 0) + indexerPods := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), defaultIndexerReplicas, false, 0) + + // Verify MC configuration for C3 cluster + verifyMCConfigForCluster(ctx, deployment, testcaseEnvInst, cfg, mcName, shPods, indexerPods) + + // Verify Monitoring Console is Ready and stays in ready state + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) + + // ################# Update Monitoring Console In Cluster Manager CR ################################## + + mcTwoName := deployment.GetName() + "-two" + cm := cfg.NewCMObject() + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, cm, deployment.GetName(), mcTwoName) + + cfg.VerifyCMReady(ctx, testcaseEnvInst, deployment) + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Deploy and verify Monitoring Console Two + mcTwo := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, mcTwoName, "") + + // ########### VERIFY MONITORING CONSOLE TWO AFTER CLUSTER MANAGER RECONFIG ################################### + VerifyMCTwoAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, true) + + // ############## VERIFY MONITORING CONSOLE ONE AFTER CLUSTER MANAGER RECONFIG ####################### + VerifyMCOneAfterCMReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, false) + + // ################# Update Monitoring Console In SHC CR ################################## + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := deployment.GetName() + "-shc" + testenv.UpdateMonitoringConsoleRefAndVerify(ctx, deployment, testcaseEnvInst, shc, shcName, mcTwoName) + + // Ensure Search Head Cluster goes to Ready Phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + if cfg.VerifyMCTwoReadyAfterSHC { + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcTwoName, mcTwo) + } + + // ############################ VERIFICATION FOR MONITORING CONSOLE TWO POST SHC RECONFIG ############################### + VerifyMCTwoAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcTwoName, shPods, indexerPods, cfg.SHCReconfigTimeout) + + // ############################ VERIFICATION FOR MONITORING CONSOLE ONE POST SHC RECONFIG ############################### + VerifyMCOneAfterSHCReconfig(ctx, deployment, testcaseEnvInst, cfg.MCReconfigParams, mcName, mc, shPods, cfg.SHCReconfigTimeout) +} + +// RunS1StandaloneAddDeleteMCTest deploys two standalone instances with a Monitoring Console, +// verifies both are registered, then deletes the second standalone and verifies the MC +// config map and peer list are updated correctly. +func RunS1StandaloneAddDeleteMCTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, standaloneOneName, standaloneTwoName string) { + mcName := deployment.GetName() + + // Deploy Standalone one with MCRef + testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, standaloneOneName, mcName) + + // Deploy MC and wait for MC to be READY + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Check Standalone is configured in MC Config Map + standalonePods := testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Add another standalone instance in namespace + testcaseEnvInst.Log.Info("Adding second standalone deployment to namespace") + standaloneTwoSpec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + }, + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("0.2"), + "memory": resource.MustParse("256Mi"), + }, + }, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standaloneTwo, err := deployment.DeployStandaloneWithGivenSpec(ctx, standaloneTwoName, standaloneTwoSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for standalone two to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, standaloneTwoName, standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check both standalones are configured in MC Config Map + standalonePods = append(standalonePods, fmt.Sprintf(testenv.StandalonePod, standaloneTwoName, 0)) + + testcaseEnvInst.Log.Info("Checking for Standalone Pod on MC Config Map after adding new standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // get revision number of the resource + resourceVersion = testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Delete standalone two and ensure MC is updated + testcaseEnvInst.Log.Info("Deleting second standalone deployment from namespace", "Standalone Name", standaloneTwoName) + deployment.GetInstance(ctx, standaloneTwoName, standaloneTwo) + err = deployment.DeleteCR(ctx, standaloneTwo) + Expect(err).To(Succeed(), "Unable to delete standalone instance", "Standalone Name", standaloneTwo) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + // Check standalone one is still configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneOneName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone One Pod in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, true) + + // Check Standalone Two NOT configured in MC Config Map + standalonePods = testenv.GeneratePodNameSlice(testenv.StandalonePod, standaloneTwoName, 1, false, 0) + + testcaseEnvInst.Log.Info("Checking for Standalone Two Pod NOT in MC Config Map after deleting second standalone") + verifyStandaloneInMC(ctx, deployment, testcaseEnvInst, standalonePods, mcName, false) +} + +// verifyStandaloneInMC verifies that the given standalone pods are present (or absent) in the +// MC config map and pod config string. +func verifyStandaloneInMC(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, pods []string, mcName string, shouldExist bool) { + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, pods, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, pods, mcName, shouldExist, false) +} + +// MCReconfigParams holds the service name and URL parameters that differ between +// V3 (master) and V4 (manager) Monitoring Console tests. +type MCReconfigParams struct { + CMServiceNameFmt string // format string for CM service name (e.g., testenv.ClusterMasterServiceName) + CMURLKey string // config map URL key (e.g., "SPLUNK_CLUSTER_MASTER_URL" or splcommon.ClusterManagerURL) +} + +// VerifyMCTwoAfterCMReconfig verifies that MC Two is correctly configured after the Cluster Manager +// has been reconfigured to point to it: CM and indexers should be present, SH should be absent. +// If checkDeployerAbsent is true, also verifies deployer is absent on MC Two (used in C3 tests). +func VerifyMCTwoAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, checkDeployerAbsent bool) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + + if checkDeployerAbsent { + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, false) + } + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC Two Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, false, false) +} + +// VerifyMCOneAfterCMReconfig verifies that MC One is correctly configured after the Cluster Manager +// has been reconfigured away from it: CM should be absent, SH should still be present. +// If checkDeployerPresent is true, also verifies deployer is still present on MC One (used in M4 tests). +func VerifyMCOneAfterCMReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, checkDeployerPresent bool) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + // CSPL-619: Indexer verification on MC One is commented out in all test variants + + if checkDeployerPresent { + testcaseEnvInst.Log.Info("Verify Deployer still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, true) + } + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config Map after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods still in MC One Config String after CM Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, true, false) +} + +// VerifyMCTwoAfterSHCReconfig verifies that MC Two has all components (CM, deployer, SH, indexers) +// after the SHC has been reconfigured to point to it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCTwoAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcTwoName string, shPods, indexerPods []string, timeout time.Duration) { + + testcaseEnvInst.Log.Info("Verify CM in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify Deployer in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcTwoName, true) + + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcTwoName, true) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads in MC two config after SHC reconfig") + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig (with wait)") + err = testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true, timeout) + Expect(err).To(Succeed(), "Timed out waiting for indexers in MC two config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcTwoName, true, false) + + testcaseEnvInst.Log.Info("Verify Indexers in MC Two Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcTwoName, true, true) + } +} + +// VerifyMCOneAfterSHCReconfig verifies that MC One has lost all components (CM, deployer, SH) +// after the SHC has been reconfigured away from it. +// If timeout > 0, uses WaitForPodsInMCConfigString; otherwise uses direct VerifyPodsInMCConfigString. +func VerifyMCOneAfterSHCReconfig(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, + params MCReconfigParams, mcName string, mc *enterpriseApi.MonitoringConsole, shPods []string, timeout time.Duration) { + + testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) + + testcaseEnvInst.Log.Info("Verify CM NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(params.CMServiceNameFmt, deployment.GetName())}, params.CMURLKey, mcName, false) + + testcaseEnvInst.Log.Info("Verify Deployer NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, + []string{fmt.Sprintf(testenv.DeployerServiceName, deployment.GetName())}, "SPLUNK_DEPLOYER_URL", mcName, false) + + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config Map after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, false) + + if timeout > 0 { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig (with wait)") + err := testcaseEnvInst.WaitForPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false, timeout) + Expect(err).To(Succeed(), "Timed out waiting for search heads to be removed from MC one config after SHC reconfig") + } else { + testcaseEnvInst.Log.Info("Verify SH Pods NOT in MC One Config String after SHC Reconfig") + testcaseEnvInst.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, false, false) + } + + // CSPL-619: Indexer verification on MC One is commented out in all test variants +} diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go deleted file mode 100644 index 6944755b4..000000000 --- a/test/secret/manager_secret_c3_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("managersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_m4_test.go b/test/secret/manager_secret_m4_test.go deleted file mode 100644 index a51dac966..000000000 --- a/test/secret/manager_secret_m4_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("managersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Manager goes to update phase - testcaseEnvInst.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/manager_secret_s1_test.go b/test/secret/manager_secret_s1_test.go deleted file mode 100644 index 58a671b67..000000000 --- a/test/secret/manager_secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("managersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("managersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Manager - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Manager to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("managersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go deleted file mode 100644 index 8baee517c..000000000 --- a/test/secret/secret_c3_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA C3", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("mastersecret, smoke, c3: secret update on indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - mcRef := deployment.GetName() - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for ClusterInitialized event to confirm cluster is fully initialized - idxcName := deployment.GetName() + "-idxc" - err = testcaseEnvInst.WaitForClusterInitialized(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for ClusterInitialized event on IndexerCluster") - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on SearchHeadCluster - shcName := deployment.GetName() + "-shc" - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Wait for PasswordSyncCompleted event on IndexerCluster - err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go deleted file mode 100644 index 3a1efe591..000000000 --- a/test/secret/secret_m4_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for M4 SVA", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - // SpecifiedTestTimeout override default timeout for m4 test cases as we have seen - // it takes more than 3000 seconds for one of the test case - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 40000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { - It("mastersecret, integration, m4: secret update on multisite indexers and search head cluster", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - siteCount := 3 - mcName := deployment.GetName() - err := deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, mcName) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF before secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Test 1 - // Update Secrets Data and - // Verify New versioned secret are created with correct value. - // Verify new secrets are mounted on pods. - // Verify New Secrets are present in server.conf (Pass4SymmKey) - // Verify New Secrets via api access (password) - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure that Cluster Master goes to update phase - testcaseEnvInst.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Verify RF SF is met - testcaseEnvInst.Log.Info("Checkin RF SF after secret change") - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each versioned secret for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Pass4SymmKey Secrets on ServerConf on MC, LM Pods - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - }) - }) -}) diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go deleted file mode 100644 index d680a4661..000000000 --- a/test/secret/secret_s1_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package secret - -import ( - "context" - "fmt" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Secret Test for SVA S1", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone deployment (S1) with LM and MC", func() { - It("mastersecret, integration, s1: Secret update on a standalone instance with LM and MC", func() { - - // Test Scenario - // 1. Update Secrets Data - // 2. Verify New versioned secret are created with correct value. - // 3. Verify new secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Update Secret Value on Secret Object - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - modifiedHecToken := testenv.GetRandomeHECToken() - modifedValue := testenv.RandomDNSName(10) - updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifedValue, modifedValue, modifedValue, modifedValue) - - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, updatedSecretData, true) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, updatedSecretData, true) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, updatedSecretData, true) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, updatedSecretData, true) - - }) - }) - - Context("Standalone deployment (S1) with LM amd MC", func() { - It("mastersecret, integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - //1. Delete Secret Object - //2. Verify New versioned secret are created with new values. - //3. Verify New secrets are mounted on pods. - //4. Verify New Secrets are present in server.conf (Pass4SymmKey) - //5. Verify New Secrets via api access (password) - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() - Expect(err).To(Succeed(), "Unable to download license file from S3") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Create standalone Deployment with License Master - mcName := deployment.GetName() - standalone, err := deployment.DeployStandaloneWithLM(ctx, deployment.GetName(), mcName) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret Object - err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for License Master to be in READY status - testcaseEnvInst.VerifyLicenseManagerReady(ctx, deployment) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) - - Context("Standalone deployment (S1)", func() { - It("mastersecret, smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { - - // Test Scenario - // 1. Delete Secret Passing Empty Data Map to secret Object - // 2. Verify New versioned secret are created with new values. - // 3. Verify New secrets are mounted on pods. - // 4. Verify New Secrets are present in server.conf (Pass4SymmKey) - // 5. Verify New Secrets via api access (password) - - // Create standalone Deployment with MonitoringConsoleRef - mcName := deployment.GetName() - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - } - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with MonitoringConsoleRef") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Deploy Monitoring Console CRD - mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // get revision number of the resource - resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) - - // Get Current Secrets Struct - namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) - testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) - Expect(err).To(Succeed(), "Unable to get secret struct") - - // Delete secret by passing empty Data Map - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) - Expect(err).To(Succeed(), "Unable to delete secret Object") - - // Ensure standalone is updating - testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // wait for custom resource resource version to change - testcaseEnvInst.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testcaseEnvInst.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) - - // Once Pods are READY check each versioned secret for updated secret keys - secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) - - // Verify Secrets on versioned secret objects - testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretStruct.Data, false) - - // Once Pods are READY check each pod for updated secret keys - verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - - // Verify secrets on pods - testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets on ServerConf on Pod - testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretStruct.Data, false) - - // Verify Hec token on InputConf on Pod - testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretStruct.Data, false) - - // Verify Secrets via api access on Pod - testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretStruct.Data, false) - }) - }) -}) diff --git a/test/secret/secret_suite_test.go b/test/secret/secret_suite_test.go index c7d40946a..b933fbbd9 100644 --- a/test/secret/secret_suite_test.go +++ b/test/secret/secret_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package secret import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 10 * ConsistentPollInterval -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "secret-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/secret/secret_test.go b/test/secret/secret_test.go new file mode 100644 index 000000000..34a32edd3 --- /dev/null +++ b/test/secret/secret_test.go @@ -0,0 +1,106 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + "github.com/splunk/splunk-operator/test/testenv" +) + +// masterManagerConfigs defines the V3 (master) and V4 (manager) variants +// shared by the C3 and M4 secret test tables. +var masterManagerConfigs = []testenv.MasterManagerTestConfig{ + {NamePrefix: "master", Label: "mastersecret", NewConfig: testenv.NewClusterReadinessConfigV3}, + {NamePrefix: "", Label: "managersecret", NewConfig: testenv.NewClusterReadinessConfigV4}, +} + +var _ = Describe("Secret test", func() { + + var testcaseEnvInst *testenv.TestCaseEnv + var deployment *testenv.Deployment + ctx := context.TODO() + + // S1 tests — both label variants use identical V4 config + s1SecretLabels := []string{"mastersecret", "managersecret"} + + for _, label := range s1SecretLabels { + label := label + Context("Standalone deployment (S1) with LM and MC", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(label+", integration, s1: Secret update on a standalone instance with LM and MC", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", integration, s1: Secret Object is recreated on delete and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteTest(ctx, deployment, testcaseEnvInst, config) + }) + + It(label+", smoke, s1: Secret Object data is repopulated in secret object on passing empty Data map and new secrets are applied to Splunk Pods", func() { + config := testenv.NewClusterReadinessConfigV4() + RunS1SecretDeleteWithMCRefTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // C3 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.Label+", smoke, c3: secret update on Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunC3SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } + + // M4 tests — V3 (master) and V4 (manager) variants + for _, tc := range masterManagerConfigs { + tc := tc + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { + BeforeEach(func() { + testenv.SpecifiedTestTimeout = 40000 + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) + + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + + It(tc.Label+", integration, m4: secret update on multisite Indexers and Search Head Cluster", func() { + config := tc.NewConfig() + RunM4SecretUpdateTest(ctx, deployment, testcaseEnvInst, config) + }) + }) + } +}) diff --git a/test/secret/secret_test_shared.go b/test/secret/secret_test_shared.go new file mode 100644 index 000000000..75bc62cb3 --- /dev/null +++ b/test/secret/secret_test_shared.go @@ -0,0 +1,260 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package secret + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" +) + +// verifySecretsPropagated checks that the given secret data has been propagated to all +// versioned secret objects, pods, server config, input config, and via the API. +func verifySecretsPropagated(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, secretData map[string][]byte, updated bool) { + // Once Pods are READY check each versioned secret for updated secret keys + secretObjectNames := testenv.GetVersionedSecretNames(testcaseEnvInst.GetName(), 2) + + // Verify Secrets on versioned secret objects + testcaseEnvInst.VerifySecretsOnSecretObjects(ctx, deployment, secretObjectNames, secretData, updated) + + // Once Pods are READY check each pod for updated secret keys + verificationPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + + // Verify secrets on pods + testcaseEnvInst.VerifySecretsOnPods(ctx, deployment, verificationPods, secretData, updated) + + // Verify Secrets on ServerConf on Pod + testcaseEnvInst.VerifySplunkServerConfSecrets(ctx, deployment, verificationPods, secretData, updated) + + // Verify Hec token on InputConf on Pod + testcaseEnvInst.VerifySplunkInputConfSecrets(deployment, verificationPods, secretData, updated) + + // Verify Secrets via api access on Pod + testcaseEnvInst.VerifySplunkSecretViaAPI(ctx, deployment, verificationPods, secretData, updated) +} + +// verifyLMAndStandaloneReady waits for License Manager then Standalone to reach READY status. +func verifyLMAndStandaloneReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, standalone *enterpriseApi.Standalone) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyLMAndClusterManagerReady waits for License Manager then Cluster Manager to reach READY status. +func verifyLMAndClusterManagerReady(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) +} + +// s1WithLMSetup holds the resources created by setupS1WithLMAndMC so that +// individual test functions can operate on them without repeating the setup. +type s1WithLMSetup struct { + standalone *enterpriseApi.Standalone + mc *enterpriseApi.MonitoringConsole + resourceVersion string + namespaceScopedSecretName string +} + +// setupS1WithLMAndMC performs the common S1 setup shared by the secret-update +// and secret-delete tests: license config map, standalone with LM, MC, and +// initial secret verification. +func setupS1WithLMAndMC(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) s1WithLMSetup { + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + mcRef := deployment.GetName() + standalone, err := config.DeployStandaloneWithLM(ctx, deployment, deployment.GetName(), mcRef) + Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") + + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, standalone) + + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err = testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + return s1WithLMSetup{ + standalone: standalone, + mc: mc, + resourceVersion: resourceVersion, + namespaceScopedSecretName: namespaceScopedSecretName, + } +} + +// verifyS1SecretChangeApplied verifies that a secret change (update or delete) +// has been applied to the S1 stack: standalone enters Updating phase, LM and +// standalone return to Ready, MC version changes, and secrets are propagated. +func verifyS1SecretChangeApplied(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, setup s1WithLMSetup, secretData map[string][]byte, updated bool) { + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + verifyLMAndStandaloneReady(ctx, deployment, testcaseEnvInst, config, setup.standalone) + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, setup.mc, setup.resourceVersion) + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretData, updated) +} + +// generateAndApplySecretUpdate creates randomized secret data and applies it to the namespace-scoped +// secret object, returning the updated data map for subsequent verification. +func generateAndApplySecretUpdate(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, namespaceScopedSecretName string) map[string][]byte { + modifiedHecToken := testenv.GetRandomHECToken() + modifiedValue := testenv.RandomDNSName(10) + updatedSecretData := testenv.GetSecretDataMap(modifiedHecToken, modifiedValue, modifiedValue, modifiedValue, modifiedValue) + err := testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, updatedSecretData) + Expect(err).To(Succeed(), "Unable to update secret Object") + return updatedSecretData +} + +// RunS1SecretUpdateTest runs the standard S1 secret update test workflow +func RunS1SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup := setupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + + // Update Secret Value on Secret Object + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, setup.namespaceScopedSecretName) + + verifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, updatedSecretData, true) +} + +// RunS1SecretDeleteTest runs the standard S1 secret delete test workflow +func RunS1SecretDeleteTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + setup := setupS1WithLMAndMC(ctx, deployment, testcaseEnvInst, config) + + // Re-fetch secret struct so we can verify its data is restored after deletion + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), setup.namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete Secret Object + err = testenv.DeleteSecretObject(ctx, deployment, testcaseEnvInst.GetName(), setup.namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + verifyS1SecretChangeApplied(ctx, deployment, testcaseEnvInst, config, setup, secretStruct.Data, false) +} + +// RunS1SecretDeleteWithMCRefTest runs the S1 secret delete test with MC reference workflow +func RunS1SecretDeleteWithMCRefTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Create standalone Deployment with MonitoringConsoleRef + mcName := deployment.GetName() + standalone := testcaseEnvInst.DeployStandaloneWithMCRef(ctx, deployment, deployment.GetName(), mcName) + + // Deploy and verify Monitoring Console + mc := testcaseEnvInst.DeployAndVerifyMonitoringConsole(ctx, deployment, deployment.GetName(), "") + + // Get revision number of the resource + resourceVersion := testcaseEnvInst.GetResourceVersion(ctx, deployment, mc) + + // Get Current Secrets Struct + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + testcaseEnvInst.Log.Info("Data in secret object", "data", secretStruct.Data) + Expect(err).To(Succeed(), "Unable to get secret struct") + + // Delete secret by passing empty Data Map + err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName, map[string][]byte{}) + Expect(err).To(Succeed(), "Unable to delete secret Object") + + // Ensure standalone is updating + testcaseEnvInst.VerifyStandalonePhase(ctx, deployment, deployment.GetName(), enterpriseApi.PhaseUpdating) + + // Wait for Standalone to be in READY status + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, secretStruct.Data, false) +} + +// RunC3SecretUpdateTest runs the standard C3 secret update test workflow +func RunC3SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + mcRef := deployment.GetName() + err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + config.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + config.VerifyC3ClusterReady(ctx, deployment, testcaseEnvInst) + + mc, resourceVersion, updatedSecretData := applySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + + // Ensure Search Head Cluster goes to Ready phase + testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on SearchHeadCluster + shcName := deployment.GetName() + "-shc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), shcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on SearchHeadCluster") + + // Ensure Indexers go to Ready phase + testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) + + // Wait for PasswordSyncCompleted event on IndexerCluster + idxcName := deployment.GetName() + "-idxc" + err = testcaseEnvInst.WaitForPasswordSyncCompleted(ctx, deployment, testcaseEnvInst.GetName(), idxcName, 2*time.Minute) + Expect(err).To(Succeed(), "Timed out waiting for PasswordSyncCompleted event on IndexerCluster") + + verifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData) +} + +// verifyPostSecretChangeCluster performs the common tail verification after a +// secret change on a clustered deployment: MC version changed, RF/SF met, and +// secrets propagated to all pods. +func verifyPostSecretChangeCluster(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, mc *enterpriseApi.MonitoringConsole, resourceVersion string, updatedSecretData map[string][]byte) { + testcaseEnvInst.VerifyMCVersionChangedAndReady(ctx, deployment, mc, resourceVersion) + + testcaseEnvInst.Log.Info("Checking RF SF after secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + verifySecretsPropagated(ctx, deployment, testcaseEnvInst, updatedSecretData, true) +} + +// applySecretUpdateAndVerifyCMUpdating deploys MC, verifies RF/SF and initial secret state, +// applies a secret update, and confirms the Cluster Manager enters the Updating phase. +// Returns the MC, its resource version, and the updated secret data for post-change verification. +func applySecretUpdateAndVerifyCMUpdating(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) (*enterpriseApi.MonitoringConsole, string, map[string][]byte) { + mc, resourceVersion := testcaseEnvInst.DeployMCAndGetVersion(ctx, deployment, deployment.GetName(), deployment.GetName()) + testcaseEnvInst.Log.Info("Checking RF SF before secret change") + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + namespaceScopedSecretName := fmt.Sprintf(testenv.NamespaceScopedSecretObjectName, testcaseEnvInst.GetName()) + _, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), namespaceScopedSecretName) + Expect(err).To(Succeed(), "Unable to get secret struct") + updatedSecretData := generateAndApplySecretUpdate(ctx, deployment, testcaseEnvInst, namespaceScopedSecretName) + config.VerifyClusterManagerPhaseUpdating(ctx, deployment, testcaseEnvInst) + return mc, resourceVersion, updatedSecretData +} + +// RunM4SecretUpdateTest runs the standard M4 secret update test workflow +func RunM4SecretUpdateTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + // Set up license config map + testenv.SetupLicenseConfigMap(ctx, testcaseEnvInst) + + siteCount := 3 + mcName := deployment.GetName() + + err := config.DeployMultisiteCluster(ctx, deployment, deployment.GetName(), 1, siteCount, mcName) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + mc, resourceVersion, updatedSecretData := applySecretUpdateAndVerifyCMUpdating(ctx, deployment, testcaseEnvInst, config) + + verifyLMAndClusterManagerReady(ctx, deployment, testcaseEnvInst, config) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + + verifyPostSecretChangeCluster(ctx, deployment, testcaseEnvInst, mc, resourceVersion, updatedSecretData) +} diff --git a/test/smartstore/manager_smartstore_test.go b/test/smartstore/manager_smartstore_test.go deleted file mode 100644 index f54a98da5..000000000 --- a/test/smartstore/manager_smartstore_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package smartstore - -import ( - "context" - "fmt" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -var _ = Describe("Smartstore test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() - var deployment *testenv.Deployment - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 5*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - }) - }) - - Context("Standalone Deployment (S1)", func() { - It("managersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("managersmartstore, smoke: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-two" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) -}) diff --git a/test/smartstore/smartstore_suite_test.go b/test/smartstore/smartstore_suite_test.go index 693ca4cf8..54e837d61 100644 --- a/test/smartstore/smartstore_suite_test.go +++ b/test/smartstore/smartstore_suite_test.go @@ -1,8 +1,20 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -10,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smartstore-" + testenv.RandomDNSName(3) @@ -26,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smartstore/smartstore_test.go b/test/smartstore/smartstore_test.go index bdd100f37..f9f598dde 100644 --- a/test/smartstore/smartstore_test.go +++ b/test/smartstore/smartstore_test.go @@ -1,363 +1,101 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package smartstore import ( "context" - "fmt" "time" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" ) +// smartstoreTestConfig extends MasterManagerTestConfig with a per-variant +// timeout used by the S1 multiple-indexes test. +type smartstoreTestConfig struct { + testenv.MasterManagerTestConfig + S1IndexesTimeout time.Duration +} + +// masterManagerSmartstoreConfigs defines the V3 (master) and V4 (manager) variants +// shared by the S1 and M4 smartstore test tables. +var masterManagerSmartstoreConfigs = []smartstoreTestConfig{ + {testenv.MasterManagerTestConfig{NamePrefix: "master", Label: "mastersmartstore", NewConfig: testenv.NewClusterReadinessConfigV3}, 2 * time.Minute}, + {testenv.MasterManagerTestConfig{NamePrefix: "", Label: "managersmartstore", NewConfig: testenv.NewClusterReadinessConfigV4}, 5 * time.Minute}, +} + var _ = Describe("Smartstore test", func() { var testcaseEnvInst *testenv.TestCaseEnv - ctx := context.TODO() var deployment *testenv.Deployment + ctx := context.TODO() - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure multiple indexes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexVolumeMap := map[string]string{"test-index-" + testenv.RandomDNSName(3): volName, - "test-index-" + testenv.RandomDNSName(3): volName, - } - testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) - - var indexSpec []enterpriseApi.IndexSpec - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - - // Create index volume spec from index volume map - for index, volume := range indexVolumeMap { - indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) - } - - // Generate smartstore spec - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volumeSpec, - IndexList: indexSpec, - } - - // Deploy Standalone - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Wait for Standalone to reach Ready phase - err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, 2*time.Minute) - Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") - - // Verify standalone goes to ready state and stays ready - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + for _, tc := range masterManagerSmartstoreConfigs { + tc := tc + Context("Standalone deployment (S1)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - for indexName := range indexVolumeMap { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Ingest data to the index - for indexName := range indexVolumeMap { - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } + It(tc.Label+", integration: Can configure multiple indexes through app", func() { + RunS1MultipleIndexesTest(ctx, deployment, testcaseEnvInst, tc.S1IndexesTimeout) + }) - // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 - for indexName := range indexVolumeMap { - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } + It(tc.Label+", integration: Can configure indexes which use default volumes through app", func() { + RunS1DefaultVolumesTest(ctx, deployment, testcaseEnvInst) + }) }) - }) - - Context("Standalone Deployment (S1)", func() { - It("mastersmartstore, integration: Can configure indexes which use default volumes through app", func() { - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, tc.NamePrefix) + }) - indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} - defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} - cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} - - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - Defaults: defaultSmartStoreSpec, - CacheManagerConf: cacheManagerSmartStoreSpec, - } - - // Deploy Standalone with given smartstore spec - standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Check index on pod - podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - - // Check special index configs - testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) - - // Ingest data to the index - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - - // Roll Hot Buckets on the test index by restarting splunk - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - - // Check for indexes on S3 - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - - // Verify Cachemanager Values - serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" - - // Validate MaxCacheSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) - - // Validate EvictionPaddingSizeMB - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) - - // Validate MaxConcurrentDownloads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) - - // Validate MaxConcurrentUploads - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) - - // Validate EvictionPolicy - testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) + It(tc.Label+", m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { + config := tc.NewConfig() + RunM4MultisiteSmartStoreTest(ctx, deployment, testcaseEnvInst, config) + }) }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4)", func() { - It("mastersmartstore, m4, integration: Can configure indexes and volumes on Multisite Indexer Cluster through app", func() { - - volName := "test-volume-" + testenv.RandomDNSName(3) - indexName := "test-index-" + testenv.RandomDNSName(3) - - volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} - indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} - smartStoreSpec := enterpriseApi.SmartStoreSpec{ - VolList: volSpec, - IndexList: indexSpec, - } - - siteCount := 3 - err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexName) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) - } - - oldBundleHash := testenv.GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") - - testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") - indexNameTwo := "test-index-" + testenv.RandomDNSName(3) - indexList := []string{indexName, indexNameTwo} - newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} - - cm := &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") - cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to add new index to cluster master") - - // Ensure that the cluster-master goes to Ready phase - testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) - - // Verify new bundle is pushed - testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) - - // Check index on pod - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - for _, index := range indexList { - testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) - } - } - - // Ingest data to the index - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenvInstance.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) - testenv.IngestFileViaMonitor(ctx, logFile, indexNameTwo, podName, deployment) - } - - // Roll Hot Buckets on the test index per indexer - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenv.RollHotToWarm(ctx, deployment, podName, indexNameTwo) - } - - // Roll index buckets and Check for indexes on S3 - for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) - testenvInstance.Log.Info("Checking index on S3", "Index Name", indexNameTwo, "Pod Name", podName) - testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexNameTwo, podName) - } - }) - }) + } Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Etc storage", func() { - - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Etc and pvc for Var volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - EtcVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageEtcEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + BeforeEach(func() { + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "master") }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, smartstore: can deploy a Standalone instance with Epehemeral Var storage", func() { - /* Test Steps - ################## SETUP #################### - * Create spec for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ############ VERIFICATION FOR STANDALONE ########### - * verify Standalone comes up with Ephemeral for Var and pvc for Etc volume - */ - - // Create App framework spec for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - VarVolumeStorageConfig: enterpriseApi.StorageClassSpec{ - StorageClassName: "TestStorageVarEph", - StorageCapacity: "1Gi", - EphemeralStorage: true, - }, - }, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") + AfterEach(func() { + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) + }) - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Etc storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageEtcEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, true) + }) + It("integration, s1, smartstore: can deploy a Standalone instance with Ephemeral Var storage", func() { + storageConfig := enterpriseApi.StorageClassSpec{StorageClassName: "TestStorageVarEph", StorageCapacity: "1Gi", EphemeralStorage: true} + RunS1EphemeralStorageTest(ctx, deployment, testcaseEnvInst, storageConfig, false) }) }) }) diff --git a/test/smartstore/smartstore_test_shared.go b/test/smartstore/smartstore_test_shared.go new file mode 100644 index 000000000..ca5981a2c --- /dev/null +++ b/test/smartstore/smartstore_test_shared.go @@ -0,0 +1,227 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package smartstore + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/gomega" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/test/testenv" + corev1 "k8s.io/api/core/v1" +) + +// RunS1MultipleIndexesTest runs the standard S1 multiple indexes SmartStore test workflow +func RunS1MultipleIndexesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, waitTimeout time.Duration) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexVolumeMap := map[string]string{ + "test-index-" + testenv.RandomDNSName(3): volName, + "test-index-" + testenv.RandomDNSName(3): volName, + } + testcaseEnvInst.Log.Info("Index secret name ", "secret name ", testcaseEnvInst.GetIndexSecretName()) + + var indexSpec []enterpriseApi.IndexSpec + volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + // Create index volume spec from index volume map + for index, volume := range indexVolumeMap { + indexSpec = append(indexSpec, testenv.GenerateIndexSpec(index, volume)) + } + + // Generate smartstore spec + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volumeSpec, + IndexList: indexSpec, + } + + // Deploy Standalone + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Wait for Standalone to reach Ready phase + err = testcaseEnvInst.WaitForStandalonePhase(ctx, deployment, testcaseEnvInst.GetName(), standalone.Name, enterpriseApi.PhaseReady, waitTimeout) + Expect(err).To(Succeed(), "Timed out waiting for Standalone to reach Ready phase") + + // Verify standalone goes to ready state and stays ready + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + for indexName := range indexVolumeMap { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } + + // Ingest data to the index + for indexName := range indexVolumeMap { + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } + + // Roll Hot Buckets on the test index by restarting splunk and check for index on S3 + for indexName := range indexVolumeMap { + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// RunS1DefaultVolumesTest runs the standard S1 default volumes SmartStore test workflow +func RunS1DefaultVolumesTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + specialConfig := map[string]int{"MaxGlobalDataSizeMB": 100, "MaxGlobalRawDataSizeMB": 100} + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + + indexSpec := []enterpriseApi.IndexSpec{{Name: indexName, RemotePath: indexName}} + defaultSmartStoreSpec := enterpriseApi.IndexConfDefaultsSpec{IndexAndGlobalCommonSpec: enterpriseApi.IndexAndGlobalCommonSpec{VolName: volName, MaxGlobalDataSizeMB: uint(specialConfig["MaxGlobalDataSizeMB"]), MaxGlobalRawDataSizeMB: uint(specialConfig["MaxGlobalRawDataSizeMB"])}} + cacheManagerSmartStoreSpec := enterpriseApi.CacheManagerSpec{MaxCacheSizeMB: 9900000, EvictionPaddingSizeMB: 1000, MaxConcurrentDownloads: 6, MaxConcurrentUploads: 6, EvictionPolicy: "lru"} + + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + Defaults: defaultSmartStoreSpec, + CacheManagerConf: cacheManagerSmartStoreSpec, + } + + // Deploy Standalone with given smartstore spec + standalone, err := deployment.DeployStandaloneWithGivenSmartStoreSpec(ctx, deployment.GetName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance ") + + // Verify standalone goes to ready state + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + + // Check index on pod + podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + + // Check special index configs + testcaseEnvInst.VerifyIndexConfigsMatch(ctx, deployment, podName, indexName, specialConfig["MaxGlobalDataSizeMB"], specialConfig["MaxGlobalRawDataSizeMB"]) + + // Ingest data to the index + logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) + testenv.CreateMockLogfile(logFile, 2000) + testenv.IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + + // Roll Hot Buckets on the test index by restarting splunk + testenv.RollHotToWarm(ctx, deployment, podName, indexName) + + // Check for indexes on S3 + testcaseEnvInst.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + + // Verify Cachemanager Values + serverConfPath := "/opt/splunk/etc/apps/splunk-operator/local/server.conf" + + // Validate MaxCacheSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_cache_size", fmt.Sprint(cacheManagerSmartStoreSpec.MaxCacheSizeMB)) + + // Validate EvictionPaddingSizeMB + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_padding", fmt.Sprint(cacheManagerSmartStoreSpec.EvictionPaddingSizeMB)) + + // Validate MaxConcurrentDownloads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_downloads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentDownloads)) + + // Validate MaxConcurrentUploads + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "max_concurrent_uploads", fmt.Sprint(cacheManagerSmartStoreSpec.MaxConcurrentUploads)) + + // Validate EvictionPolicy + testcaseEnvInst.VerifyConfOnPod(deployment, podName, serverConfPath, "eviction_policy", cacheManagerSmartStoreSpec.EvictionPolicy) +} + +// RunS1EphemeralStorageTest deploys a Standalone with one ephemeral storage volume configured and verifies it is ready. +// Pass etcStorage=true to set EtcVolumeStorageConfig, false to set VarVolumeStorageConfig. +func RunS1EphemeralStorageTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, storageConfig enterpriseApi.StorageClassSpec, etcStorage bool) { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + }, + } + if etcStorage { + spec.CommonSplunkSpec.EtcVolumeStorageConfig = storageConfig + } else { + spec.CommonSplunkSpec.VarVolumeStorageConfig = storageConfig + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App Framework") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) +} + +// verifyM4ClusterAndRFSF verifies cluster manager and multisite cluster are ready and RF/SF is met. +func verifyM4ClusterAndRFSF(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig, siteCount int) { + config.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + testcaseEnvInst.VerifyM4ComponentsReady(ctx, deployment, siteCount) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) +} + +// RunM4MultisiteSmartStoreTest runs the standard M4 multisite SmartStore test workflow +func RunM4MultisiteSmartStoreTest(ctx context.Context, deployment *testenv.Deployment, testcaseEnvInst *testenv.TestCaseEnv, config *testenv.ClusterReadinessConfig) { + volName := "test-volume-" + testenv.RandomDNSName(3) + indexName := "test-index-" + testenv.RandomDNSName(3) + + volSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volName, testenv.GetS3Endpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultS3Region())} + indexSpec := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexName, volName)} + smartStoreSpec := enterpriseApi.SmartStoreSpec{ + VolList: volSpec, + IndexList: indexSpec, + } + + siteCount := 3 + err := config.DeployMultisiteClusterWithIndexes(ctx, deployment, deployment.GetName(), 1, siteCount, testcaseEnvInst.GetIndexSecretName(), smartStoreSpec) + Expect(err).To(Succeed(), "Unable to deploy cluster") + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Use multisite workflow helper to verify index, ingest data, roll to warm, and verify on S3 + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexName) + + // Get old bundle hash before adding new index + oldBundleHash := config.GetBundleHash(ctx, deployment) + + testcaseEnvInst.Log.Info("Adding new index to Cluster Manager CR") + indexNameTwo := "test-index-" + testenv.RandomDNSName(3) + indexList := []string{indexName, indexNameTwo} + newIndex := []enterpriseApi.IndexSpec{testenv.GenerateIndexSpec(indexNameTwo, volName)} + + // Update CR with new index based on API version + config.AppendSmartStoreIndex(ctx, deployment, newIndex) + + verifyM4ClusterAndRFSF(ctx, deployment, testcaseEnvInst, config, siteCount) + + // Verify new bundle is pushed (only for v3) + if config.APIVersion == "v3" { + testcaseEnvInst.VerifyClusterManagerBundlePush(ctx, deployment, testcaseEnvInst.GetName(), 1, oldBundleHash) + } + + // Verify both indexes on all sites + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), siteNumber, 0) + for _, index := range indexList { + testcaseEnvInst.VerifyIndexFoundOnPod(ctx, deployment, podName, index) + } + } + + // Use multisite workflow helper for the new index + testcaseEnvInst.Log.Info("Ingesting data on index", "Index Name", indexNameTwo) + testcaseEnvInst.MultisiteIndexerWorkflow(ctx, deployment, deployment.GetName(), siteCount, indexNameTwo) +} diff --git a/test/smoke/cluster_manager_sites_response.go b/test/smoke/cluster_manager_sites_response.go deleted file mode 100644 index f9bcdee60..000000000 --- a/test/smoke/cluster_manager_sites_response.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package smoke - -// ClusterManagerSitesResponse is a representation of the sites managed by a Splunk cluster-manager -// Endpoint: /services/cluster/manager/sites -type ClusterManagerSitesResponse struct { - Entries []ClusterManagerSitesEntry `json:"entry"` -} - -// ClusterManagerSitesEntry represents a site of an indexer cluster with its metadata -type ClusterManagerSitesEntry struct { - Name string `json:"name"` - Content ClusterManagerSitesContent `json:"content"` -} - -// ClusterManagerSitesContent represents detailed information about a site -type ClusterManagerSitesContent struct { - Peers map[string]ClusterManagerSitesPeer `json:"peers"` -} - -// ClusterManagerSitesPeer reprensents an indexer peer member of a site -type ClusterManagerSitesPeer struct { - ServerName string `json:"server_name"` -} diff --git a/test/smoke/smoke_suite_test.go b/test/smoke/smoke_suite_test.go index b47f8a984..f0ee154dc 100644 --- a/test/smoke/smoke_suite_test.go +++ b/test/smoke/smoke_suite_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package smoke import ( "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,15 +22,6 @@ import ( "github.com/splunk/splunk-operator/test/testenv" ) -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - var ( testenvInstance *testenv.TestEnv testSuiteName = "smoke-" + testenv.RandomDNSName(3) @@ -39,7 +29,6 @@ var ( // TestBasic is the main entry point func TestBasic(t *testing.T) { - RegisterFailHandler(Fail) RunSpecs(t, "Running "+testSuiteName) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index d6d969819..66f355c33 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,16 +15,10 @@ package smoke import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/types" - . "github.com/onsi/gomega" "github.com/splunk/splunk-operator/test/testenv" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - corev1 "k8s.io/api/core/v1" ) var _ = Describe("Smoke test", func() { @@ -34,134 +28,41 @@ var _ = Describe("Smoke test", func() { ctx := context.TODO() BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - - // Validate test prerequisites early to fail fast - err = testcaseEnvInst.ValidateTestPrerequisites(ctx, deployment) - Expect(err).To(Succeed(), "Test prerequisites validation failed") + testcaseEnvInst, deployment = testenv.SetupTestCaseEnv(testenvInstance, "") }) AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if types.SpecState(CurrentSpecReport().State) == types.SpecStateFailed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } + testenv.TeardownTestCaseEnv(testcaseEnvInst, deployment) }) Context("Standalone deployment (S1)", func() { It("smoke, basic, s1: can deploy a standalone instance", func() { - - standalone, err := deployment.DeployStandalone(ctx, deployment.GetName(), "", "") - Expect(err).To(Succeed(), "Unable to deploy standalone instance ") - - // Verify standalone goes to ready state - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) + testenv.RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName()) }) }) - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { + Context("Clustered deployment (C3 - Clustered Indexer, Search Head Cluster)", func() { It("smoke, basic, c3: can deploy indexers and search head cluster", func() { - - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), 3, true /*shc*/, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure Search Head Cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Ensure Indexers go to Ready phase - testcaseEnvInst.VerifySingleSiteIndexersReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 3, "") }) }) - Context("Multisite cluster deployment (M4 - Multisite indexer cluster, Search head cluster)", func() { + Context("Multisite cluster deployment (M4 - Multisite Indexer Cluster, Search Head Cluster)", func() { It("smoke, basic, m4: can deploy indexers and search head cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteClusterWithSearchHead(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Ensure search head cluster go to Ready phase - testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM4DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3, "") }) }) - Context("Multisite cluster deployment (M1 - multisite indexer cluster)", func() { + Context("Multisite cluster deployment (M1 - Multisite Indexer Cluster)", func() { It("smoke, basic: can deploy multisite indexers cluster", func() { - - siteCount := 3 - err := deployment.DeployMultisiteCluster(ctx, deployment.GetName(), 1, siteCount, "") - Expect(err).To(Succeed(), "Unable to deploy cluster") - - // Ensure that the cluster-manager goes to Ready phase - testcaseEnvInst.VerifyClusterManagerReady(ctx, deployment) - - // Ensure the indexers of all sites go to Ready phase - testcaseEnvInst.VerifyIndexersReady(ctx, deployment, siteCount) - - // Ensure cluster configured as multisite - testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) - - // Verify RF SF is met - testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + testenv.RunM1DeploymentWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), 1, 3) }) }) Context("Standalone deployment (S1) with Service Account", func() { It("smoke, basic, s1: can deploy a standalone instance attached to a service account", func() { - // Create Service Account serviceAccountName := "smoke-service-account" - testcaseEnvInst.CreateServiceAccount(serviceAccountName) - - standaloneSpec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - Image: testcaseEnvInst.GetSplunkImage(), - }, - Volumes: []corev1.Volume{}, - ServiceAccount: serviceAccountName, - }, - } - - // Create standalone Deployment with License Manager - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), standaloneSpec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with LM") - - // Wait for Standalone to be in READY status - testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, deployment.GetName(), standalone) - - // Verify serviceAccount is configured on Pod - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + testenv.RunStandaloneWithServiceAccountWorkflow(ctx, deployment, testcaseEnvInst, deployment.GetName(), serviceAccountName) }) }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index a60554bd6..6b6f08ef5 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -109,7 +109,7 @@ var PVTestAppsLocation = "appframework/100mb_sample_apps/" // AppStagingLocOnPod is the volume on Splunk pod where apps will be copied from operator var AppStagingLocOnPod = "/operator-staging/appframework/" -// AppDownloadVolume location on Operator pod for App framework downloads +// AppDownloadVolume location on Operator pod for App Framework downloads var AppDownloadVolume = "/opt/splunk/appframework/downloadedApps/" // GenerateAppSourceSpec return AppSourceSpec struct with given values @@ -125,7 +125,7 @@ func GenerateAppSourceSpec(appSourceName string, appSourceLocation string, appSo func GetPodAppStatus(ctx context.Context, deployment *Deployment, podName string, ns string, appname string, clusterWideInstall bool) (string, string, error) { // For clusterwide install do not check for versions on deployer and cluster-manager as the apps arent installed there if clusterWideInstall && (strings.Contains(podName, "-cluster-manager-") || strings.Contains(podName, splcommon.TestClusterManagerDashed) || strings.Contains(podName, "-deployer-")) { - logf.Log.Info("Pod skipped as install is Cluter-wide", "PodName", podName) + logf.Log.Info("Pod skipped as install is Cluter-wide", "podName", podName) return "", "", nil } output, err := GetPodAppInstallStatus(ctx, deployment, podName, ns, appname) @@ -154,11 +154,11 @@ func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string } } filePath := fmt.Sprintf("/opt/splunk/%s/%s/default/app.conf", path, appname) - logf.Log.Info("Check app version", "App", appname, "Conf file", filePath) + logf.Log.Info("Check app version", "app", appname, "confFile", filePath) confline, err := GetConfLineFromPod(podName, filePath, ns, "version", "launcher", true) if err != nil { - logf.Log.Error(err, "Failed to get version from pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get version from pod", "podName", podName) return "", err } version := strings.TrimSpace(strings.Split(confline, "=")[1]) @@ -181,7 +181,7 @@ func GetPodAppInstallStatus(ctx context.Context, deployment *Deployment, podName return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout) return strings.TrimSuffix(stdout, "\n"), nil } @@ -195,7 +195,7 @@ func GetPodAppbtoolStatus(ctx context.Context, deployment *Deployment, podName s logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) if len(stdout) > 0 { if strings.Contains(strings.Split(stdout, "\n")[0], "App is disabled") { @@ -218,13 +218,13 @@ func GetAppFileList(appList []string) []string { // GetAppframeworkManualUpdateConfigMap gets config map for given manual update configmap func GetAppframeworkManualUpdateConfigMap(ctx context.Context, deployment *Deployment, ns string) (*corev1.ConfigMap, error) { ConfigMapName := fmt.Sprintf(AppframeworkManualUpdateConfigMap, ns) - logf.Log.Info("Get config map for", "CONFIG MAP NAME", ConfigMapName) + logf.Log.Info("Get config map for", "configMapName", ConfigMapName) ConfigMap, err := GetConfigMap(ctx, deployment, ns, ConfigMapName) if err != nil { logf.Log.Error(err, "Failed to get splunk manual poll Config Map") return ConfigMap, err } - logf.Log.Info("Config Map contents", "CONFIG MAP NAME", ConfigMapName, "Data", ConfigMap.Data) + logf.Log.Info("Config Map contents", "configMapName", ConfigMapName, "data", ConfigMap.Data) return ConfigMap, err } @@ -234,19 +234,19 @@ func GetAppDeploymentInfoStandalone(ctx context.Context, deployment *Deployment, appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, standalone) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := standalone.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Standalone AppInfo Struct", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Standalone Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "standaloneName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -256,19 +256,19 @@ func GetAppDeploymentInfoIngestorCluster(ctx context.Context, deployment *Deploy appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, ingestor) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := ingestor.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Ingestor AppInfo Struct", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Ingestor Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "ingestorName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -278,19 +278,19 @@ func GetAppDeploymentInfoMonitoringConsole(ctx context.Context, deployment *Depl appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, mc) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := mc.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Monitoring Console AppInfo Struct", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Monitoring Console Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "monitoringConsoleName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -300,41 +300,41 @@ func GetAppDeploymentInfoClusterManager(ctx context.Context, deployment *Deploym appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Manager AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Manager Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterManagerName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } -// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Manager, appSourceName and appName +// GetAppDeploymentInfoClusterMaster returns AppDeploymentInfo for given Cluster Master, appSourceName and appName func GetAppDeploymentInfoClusterMaster(ctx context.Context, deployment *Deployment, testenvInstance *TestCaseEnv, name string, appSourceName string, appName string) (enterpriseApi.AppDeploymentInfo, error) { cm := &enterpriseApiV3.ClusterMaster{} appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Cluster Master AppInfo Struct", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Cluster Master Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "clusterMasterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -344,19 +344,19 @@ func GetAppDeploymentInfoSearchHeadCluster(ctx context.Context, deployment *Depl appDeploymentInfo := enterpriseApi.AppDeploymentInfo{} err := deployment.GetInstance(ctx, name, cm) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name) return appDeploymentInfo, err } appInfoList := cm.Status.AppContext.AppsSrcDeployStatus[appSourceName].AppDeploymentInfoList for _, appInfo := range appInfoList { - testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("Checking Search Head Cluster AppInfo Struct", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) if strings.Contains(appName, appInfo.AppName) { - testenvInstance.Log.Info("App Deployment Info found.", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "AppDeploymentInfo", appInfo) + testenvInstance.Log.Info("App Deployment Info found.", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appDeploymentInfo", appInfo) appDeploymentInfo = appInfo return appDeploymentInfo, nil } } - testenvInstance.Log.Info("App Info not found in App Info List", "App Name", appName, "App Source", appSourceName, "Search Head Name Name", name, "App Info List", appInfoList) + testenvInstance.Log.Info("App Info not found in App Info List", "appName", appName, "appSource", appSourceName, "searchHeadClusterName", name, "appInfoList", appInfoList) return appDeploymentInfo, err } @@ -389,7 +389,7 @@ func GetAppDeploymentInfo(ctx context.Context, deployment *Deployment, testenvIn func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeName string, scope string, appSourceName string, s3TestDir string, pollInterval int) enterpriseApi.AppFrameworkSpec { var volumeSpec []enterpriseApi.VolumeSpec - // Create App framework volume + // Create App Framework volume switch ClusterProvider { case "eks": volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3", GetDefaultS3Region())} @@ -404,7 +404,7 @@ func GenerateAppFrameworkSpec(ctx context.Context, testenvInstance *TestCaseEnv, volumeSpec = []enterpriseApi.VolumeSpec{GenerateIndexVolumeSpec(volumeName, GetGCPEndpoint(), testenvInstance.GetIndexSecretName(), "gcp", "gcs", GetDefaultS3Region())} default: - testenvInstance.Log.Info("Failed to identify cluster provider name: Should be 'eks' or 'azure' or 'gcp' ") + testenvInstance.Log.Info("Failed to identify provider: Should be 'eks' or 'azure' or 'gcp'") } // AppSourceDefaultSpec: Remote Storage volume name and Scope of App deployment @@ -601,7 +601,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.Standalone{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -609,7 +609,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.MonitoringConsole{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -617,7 +617,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.SearchHeadCluster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -625,7 +625,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.ClusterMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -633,7 +633,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.ClusterManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -641,7 +641,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApiV3.LicenseMaster{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress @@ -649,7 +649,7 @@ func GetIsDeploymentInProgressFlag(ctx context.Context, deployment *Deployment, cr := &enterpriseApi.LicenseManager{} err := deployment.GetInstance(ctx, name, cr) if err != nil { - testenvInstance.Log.Error(err, "Failed to get CR ", "CR Name", name, "CR Kind", crKind) + testenvInstance.Log.Error(err, "Failed to get CR ", "crName", name, "crKind", crKind) return isDeploymentInProgress, err } isDeploymentInProgress = cr.Status.AppContext.IsDeploymentInProgress diff --git a/test/testenv/assertion_helpers.go b/test/testenv/assertion_helpers.go new file mode 100644 index 000000000..c69a92b12 --- /dev/null +++ b/test/testenv/assertion_helpers.go @@ -0,0 +1,198 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// ScaleSearchHeadCluster scales a Search Head Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleSearchHeadCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + shcName := deploymentName + "-shc" + + // Get instance of current SHC CR with latest config + shc := &enterpriseApi.SearchHeadCluster{} + GetInstanceWithExpect(ctx, deployment, shc, shcName, "Failed to get instance of Search Head Cluster") + + // Update Replicas of SHC + shc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, shc, "Failed to scale Search Head Cluster") + + // Verify Search Head Cluster scales up and goes to ScalingUp phase + testcaseenv.VerifySearchHeadClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp) +} + +// ScaleIndexerCluster scales an Indexer Cluster to the specified replica count +func (testcaseenv *TestCaseEnv) ScaleIndexerCluster(ctx context.Context, deployment *Deployment, deploymentName string, newReplicas int) { + idxcName := deploymentName + "-idxc" + + // Get instance of current Indexer CR with latest config + idxc := &enterpriseApi.IndexerCluster{} + GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Failed to get instance of Indexer Cluster") + + // Update Replicas of Indexer Cluster + idxc.Spec.Replicas = int32(newReplicas) + UpdateCRWithExpect(ctx, deployment, idxc, "Failed to scale Indexer Cluster") + + // Verify Indexer Cluster scales up and goes to ScalingUp phase + testcaseenv.VerifyIndexerClusterPhase(ctx, deployment, enterpriseApi.PhaseScalingUp, idxcName) +} + +// UpdateMonitoringConsoleRef updates the MonitoringConsoleRef in a CR and waits for the change to apply +func UpdateMonitoringConsoleRefAndVerify(ctx context.Context, deployment *Deployment, testcaseenv *TestCaseEnv, obj interface{}, instanceName string, newMCName string) { + // Get current resource version before update + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, obj) + + // Update the MonitoringConsoleRef based on the type + switch cr := obj.(type) { + case *enterpriseApi.ClusterManager: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + case *enterpriseApiV3.ClusterMaster: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + case *enterpriseApi.SearchHeadCluster: + GetInstanceWithExpect(ctx, deployment, cr, instanceName, "Failed to get instance") + cr.Spec.MonitoringConsoleRef.Name = newMCName + UpdateCRWithExpect(ctx, deployment, cr, "Failed to update MonitoringConsoleRef") + } + + // Wait for custom resource version to change + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, obj, resourceVersion) +} + +// VerifyMCConfigForC3Cluster verifies the standard MC configuration for a C3 cluster +func (testcaseenv *TestCaseEnv) VerifyMCConfigForC3Cluster(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shReplicas int, indexerReplicas int, shouldExist bool) { + // Check Cluster Manager in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(ClusterMasterServiceName, deploymentName)}, "SPLUNK_CLUSTER_MASTER_URL", mcName, shouldExist) + + // Check Deployer in MC Config Map + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{fmt.Sprintf(DeployerServiceName, deploymentName)}, "SPLUNK_DEPLOYER_URL", mcName, shouldExist) + + // Check Search Head pods in MC Config Map + shPods := GeneratePodNameSlice(SearchHeadPod, deploymentName, shReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, shPods, "SPLUNK_SEARCH_HEAD_URL", mcName, shouldExist) + + // Check Search Heads in MC Pod config string + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, shPods, mcName, shouldExist, false) + + // Check Indexers in MC Pod config string + indexerPods := GeneratePodNameSlice(IndexerPod, deploymentName, indexerReplicas, false, 0) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, indexerPods, mcName, shouldExist, true) +} + +// DeployAndVerifyC3WithMC deploys a C3 cluster with a given MC and verifies all components are ready +func (testcaseenv *TestCaseEnv) DeployAndVerifyC3WithMC(ctx context.Context, deployment *Deployment, deploymentName string, indexerReplicas int, mcName string) { + err := deployment.DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx, deploymentName, indexerReplicas, true, mcName) + Expect(err).To(Succeed(), "Unable to deploy Cluster Master") + + // Verify all components are ready + testcaseenv.VerifyClusterMasterReady(ctx, deployment) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// DeployStandaloneWithMCRef deploys a standalone instance with a MonitoringConsoleRef +func (testcaseenv *TestCaseEnv) DeployStandaloneWithMCRef(ctx context.Context, deployment *Deployment, deploymentName string, mcName string) *enterpriseApi.Standalone { + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseenv.GetSplunkImage(), + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deploymentName, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone instance") + + // Wait for Standalone to be in READY status + testcaseenv.VerifyStandaloneReady(ctx, deployment, deploymentName, standalone) + + return standalone +} + +// VerifyStandaloneInMC verifies that a standalone instance is configured in the MC +func (testcaseenv *TestCaseEnv) VerifyStandaloneInMC(ctx context.Context, deployment *Deployment, deploymentName string, mcName string, shouldExist bool) { + standalonePod := fmt.Sprintf(StandalonePod, deploymentName, 0) + testcaseenv.VerifyPodsInMCConfigMap(ctx, deployment, []string{standalonePod}, "SPLUNK_STANDALONE_URL", mcName, shouldExist) + testcaseenv.VerifyPodsInMCConfigString(ctx, deployment, []string{standalonePod}, mcName, shouldExist, false) +} + +// VerifyLMConfiguredOnPods verifies License Manager is configured on all given pods +func VerifyLMConfiguredOnPods(ctx context.Context, deployment *Deployment, podNames []string) { + for _, podName := range podNames { + VerifyLMConfiguredOnPod(ctx, deployment, podName) + } +} + +// IngestDataOnIndexers ingests test data on all indexer pods +func IngestDataOnIndexers(ctx context.Context, deployment *Deployment, deploymentName string, indexerCount int) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, DefaultIngestIndex, podName, deployment) + } +} + +// VerifyM1ClusterReady verifies the cluster coordinator, indexers, and multisite status are ready (no SHC). +func (testcaseenv *TestCaseEnv) VerifyM1ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) +} + +// VerifyM4ClusterReady verifies the cluster coordinator, indexers, multisite status, and SHC are ready. +func (testcaseenv *TestCaseEnv) VerifyM4ClusterReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyM4IndexersAndSHCReady verifies the cluster coordinator, indexers, and SHC are ready (without multisite check). +func (testcaseenv *TestCaseEnv) VerifyM4IndexersAndSHCReady(ctx context.Context, deployment *Deployment, siteCount int, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyC3ClusterReady verifies the cluster coordinator, SHC, and single-site indexers are ready. +func (testcaseenv *TestCaseEnv) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, verifyCoordinator func(context.Context, *Deployment)) { + verifyCoordinator(ctx, deployment) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) + testcaseenv.VerifySingleSiteIndexersReady(ctx, deployment) +} + +// IngestDataOnMultisiteIndexers ingests test data on all multisite indexer pods +func IngestDataOnMultisiteIndexers(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int) { + for site := 1; site <= siteCount; site++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, site, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, DefaultIngestIndex, podName, deployment) + } +} diff --git a/test/testenv/azureutils.go b/test/testenv/azureutils.go index 84a3f0601..04491d808 100644 --- a/test/testenv/azureutils.go +++ b/test/testenv/azureutils.go @@ -338,7 +338,7 @@ func (client *AzureBlobClient) DownloadFileFromAzure(ctx context.Context, downlo return "", err } - logf.Log.Info("Download from Azure successful:", "File", downloadRequest.RemoteFile) + logf.Log.Info("Download from Azure successful:", "file", downloadRequest.RemoteFile) return localFile.Name(), err } @@ -354,7 +354,7 @@ func DownloadFilesFromAzure(ctx context.Context, endPoint, accountKey, accountNa } _, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, endPoint, StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -374,7 +374,7 @@ func DownloadLicenseFromAzure(ctx context.Context, downloadDir string) (string, azureBlobClient := &AzureBlobClient{} filename, err := azureBlobClient.DownloadFileFromAzure(ctx, downloadRequest, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount) if err != nil { - logf.Log.Error(err, "Unable to download license file", "File", filename) + logf.Log.Error(err, "Unable to download license file", "file", filename) } return filename, err } @@ -431,7 +431,7 @@ func UploadFilesToAzure(ctx context.Context, accountName, accountKey, uploadFrom fileFullPath := "https://" + StorageAccount + ".blob.core.windows.net" + "/" + azureIndexesContainer + "/" + containerName + "/" + key fileName, err := UploadFileToAzure(ctx, accountName, accountKey, fileFullPath, fileLocation) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } uploadedFiles = append(uploadedFiles, fileName) diff --git a/test/testenv/cmutil.go b/test/testenv/cmutil.go index a1b71f3ce..e9b251ffb 100644 --- a/test/testenv/cmutil.go +++ b/test/testenv/cmutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ type ClusterManagerSitesContent struct { Peers map[string]ClusterManagerSitesPeer `json:"peers"` } -// ClusterManagerSitesPeer reprensents an indexer peer member of a site +// ClusterManagerSitesPeer represents an indexer peer member of a site type ClusterManagerSitesPeer struct { ServerName string `json:"server_name"` } @@ -158,7 +158,7 @@ func CheckIndexerOnCM(ctx context.Context, deployment *Deployment, indexerName s restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "peer") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Peer found On CM", "Indexer Name", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Peer found On CM", "indexerName", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == indexerName { found = true break @@ -172,7 +172,7 @@ func CheckSearchHeadOnCM(ctx context.Context, deployment *Deployment, searchHead restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") found := false for _, entry := range restResponse.Entry { - logf.Log.Info("Search Head On CM", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Head On CM", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Label == searchHeadName { found = true break @@ -186,7 +186,7 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { restResponse := GetIndexersOrSearchHeadsOnCM(ctx, deployment, "sh") searchHeadRemoved := true for _, entry := range restResponse.Entry { - logf.Log.Info("Search Found", "Search Head", entry.Content.Label, "Status", entry.Content.Status) + logf.Log.Info("Search Found", "searchHead", entry.Content.Label, "status", entry.Content.Status) if entry.Content.Status == "Disconnected" { searchHeadRemoved = false } @@ -194,21 +194,6 @@ func CheckSearchHeadRemoved(ctx context.Context, deployment *Deployment) bool { return searchHeadRemoved } -// RollHotBuckets roll hot buckets in cluster -func RollHotBuckets(ctx context.Context, deployment *Deployment) bool { - podName := fmt.Sprintf("splunk-%s-%s-0", deployment.GetName(), "cluster-manager") - stdin := "/opt/splunk/bin/splunk rolling-restart cluster-peers -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - - return strings.Contains(stdout, "Rolling restart of all cluster peers has been initiated.") -} - // ClusterManagerInfoEndpointResponse is represtentation of /services/cluster/manager/info endpoint type ClusterManagerInfoEndpointResponse struct { Entry []struct { @@ -275,7 +260,7 @@ func CMBundlePushstatus(ctx context.Context, deployment *Deployment, previousBun // Check if new bundle was pushed by comparing hash if previousBundleHash != "" { if entry.Content.BundleID == previousBundleHash { - logf.Log.Info("Bundle hash not updated", "old Bundle hash", previousBundleHash, "new Bundle hash", entry.Content.BundleID) + logf.Log.Info("Bundle hash not updated", "oldBundleHash", previousBundleHash, "newBundleHash", entry.Content.BundleID) continue } } @@ -298,6 +283,6 @@ func GetClusterManagerBundleHash(ctx context.Context, deployment *Deployment, cr restResponse := ClusterManagerInfoResponse(ctx, deployment, podName) bundleHash := restResponse.Entry[0].Content.ActiveBundle.Checksum - logf.Log.Info("Bundle Hash on Cluster Manager Found", "Hash", bundleHash) + logf.Log.Info("Bundle Hash on Cluster Manager Found", "hash", bundleHash) return bundleHash } diff --git a/test/testenv/common_test_patterns.go b/test/testenv/common_test_patterns.go new file mode 100644 index 000000000..05453692b --- /dev/null +++ b/test/testenv/common_test_patterns.go @@ -0,0 +1,288 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + corev1 "k8s.io/api/core/v1" +) + +// ClusterReadinessConfig holds v3/v4 API version callbacks for cluster and License Manager +// readiness verification. Shared across test packages to avoid per-package duplication. +type ClusterReadinessConfig struct { + LicenseManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + ClusterManagerReady func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) + APIVersion string +} + +// MasterManagerTestConfig pairs a name prefix and test label with a factory +// function that returns the appropriate ClusterReadinessConfig. +// This is the standard config type shared by test packages that loop over +// V3 (master) and V4 (manager) variants. +type MasterManagerTestConfig struct { + NamePrefix string + Label string + NewConfig func() *ClusterReadinessConfig +} + +// NewClusterReadinessConfigV3 creates a ClusterReadinessConfig for v3 API (LicenseMaster/ClusterMaster) +func NewClusterReadinessConfigV3() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseMasterReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterMasterReady(ctx, deployment) + }, + APIVersion: "v3", + } +} + +// NewClusterReadinessConfigV4 creates a ClusterReadinessConfig for v4 API (LicenseManager/ClusterManager) +func NewClusterReadinessConfigV4() *ClusterReadinessConfig { + return &ClusterReadinessConfig{ + LicenseManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyLicenseManagerReady(ctx, deployment) + }, + ClusterManagerReady: func(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyClusterManagerReady(ctx, deployment) + }, + APIVersion: "v4", + } +} + +// DeployStandaloneWithLM deploys a standalone with the appropriate License Manager type for +// the API version: LicenseMaster (v3) or LicenseManager (v4). +func (c *ClusterReadinessConfig) DeployStandaloneWithLM(ctx context.Context, deployment *Deployment, name, mcRef string) (*enterpriseApi.Standalone, error) { + if c.APIVersion == "v3" { + return deployment.DeployStandaloneWithLMaster(ctx, name, mcRef) + } + return deployment.DeployStandaloneWithLM(ctx, name, mcRef) +} + +// DeployMultisiteCluster deploys a multisite cluster with the appropriate Cluster Manager type +// for the API version: ClusterMaster (v3) or ClusterManager (v4). +func (c *ClusterReadinessConfig) DeployMultisiteCluster(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, mcRef string) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + } + return deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) +} + +// VerifyC3ClusterReady verifies the C3 cluster is ready using the config's ClusterManagerReady callback. +func (c *ClusterReadinessConfig) VerifyC3ClusterReady(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + testcaseEnv.VerifyC3ClusterReady(ctx, deployment, func(ctx context.Context, d *Deployment) { + c.ClusterManagerReady(ctx, d, testcaseEnv) + }) +} + +// VerifyClusterManagerPhaseUpdating asserts the Cluster Manager (or ClusterMaster for v3) +// has entered the Updating phase. +func (c *ClusterReadinessConfig) VerifyClusterManagerPhaseUpdating(ctx context.Context, deployment *Deployment, testcaseEnv *TestCaseEnv) { + if c.APIVersion == "v3" { + testcaseEnv.VerifyClusterMasterPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } else { + testcaseEnv.VerifyClusterManagerPhase(ctx, deployment, enterpriseApi.PhaseUpdating) + } +} + +// ClusterManagerPVCType returns the PVC label fragment for the Cluster Manager: +// "cluster-master" for v3, "cluster-manager" for v4. +func (c *ClusterReadinessConfig) ClusterManagerPVCType() string { + if c.APIVersion == "v3" { + return "cluster-master" + } + return "cluster-manager" +} + +// DeleteClusterManager fetches and deletes the Cluster Manager CR for the appropriate API version. +func (c *ClusterReadinessConfig) DeleteClusterManager(ctx context.Context, deployment *Deployment) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Master instance", "Cluster Master Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Master instance", "Cluster Master Name", cm) + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Unable to GET Cluster Manager instance", "Cluster Manager Name", cm) + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager instance", "Cluster Manager Name", cm) + } +} + +// DeployMultisiteClusterWithIndexes deploys a multisite cluster with SmartStore indexes using +// the appropriate Cluster Manager type for the API version. +func (c *ClusterReadinessConfig) DeployMultisiteClusterWithIndexes(ctx context.Context, deployment *Deployment, name string, indexerReplicas, siteCount int, secretName string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { + if c.APIVersion == "v3" { + return deployment.DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) + } + return deployment.DeployMultisiteClusterWithSearchHeadAndIndexes(ctx, name, indexerReplicas, siteCount, secretName, smartStoreSpec) +} + +// GetBundleHash returns the current bundle hash for the Cluster Manager (or ClusterMaster for v3). +func (c *ClusterReadinessConfig) GetBundleHash(ctx context.Context, deployment *Deployment) string { + if c.APIVersion == "v3" { + return GetClusterManagerBundleHash(ctx, deployment, "ClusterMaster") + } + return GetClusterManagerBundleHash(ctx, deployment, "ClusterManager") +} + +// AppendSmartStoreIndex appends a new SmartStore index to the Cluster Manager CR +// for the appropriate API version. +func (c *ClusterReadinessConfig) AppendSmartStoreIndex(ctx context.Context, deployment *Deployment, newIndex []enterpriseApi.IndexSpec) { + name := deployment.GetName() + if c.APIVersion == "v3" { + cm := &enterpriseApiV3.ClusterMaster{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Master") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to Cluster Master") + } else { + cm := &enterpriseApi.ClusterManager{} + err := deployment.GetInstance(ctx, name, cm) + Expect(err).To(Succeed(), "Failed to get instance of Cluster Manager") + cm.Spec.SmartStore.IndexList = append(cm.Spec.SmartStore.IndexList, newIndex...) + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed to add new index to Cluster Manager") + } +} + +// DeployMCAndGetVersion deploys and verifies a Monitoring Console, then returns both the MC +// instance and its current resource version. +func (testcaseenv *TestCaseEnv) DeployMCAndGetVersion(ctx context.Context, deployment *Deployment, name string, lmRef string) (*enterpriseApi.MonitoringConsole, string) { + mc := testcaseenv.DeployAndVerifyMonitoringConsole(ctx, deployment, name, lmRef) + resourceVersion := testcaseenv.GetResourceVersion(ctx, deployment, mc) + return mc, resourceVersion +} + +// DeployAndVerifyStandalone deploys a standalone instance and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyStandalone(ctx context.Context, deployment *Deployment, name string, mcRef string, licenseManagerRef string) *enterpriseApi.Standalone { + standalone, err := deployment.DeployStandalone(ctx, name, mcRef, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") + + testcaseenv.VerifyStandaloneReady(ctx, deployment, name, standalone) + return standalone +} + +// DeployAndVerifyMonitoringConsole deploys a Monitoring Console and verifies it reaches ready state +func (testcaseenv *TestCaseEnv) DeployAndVerifyMonitoringConsole(ctx context.Context, deployment *Deployment, name string, licenseManagerRef string) *enterpriseApi.MonitoringConsole { + mc, err := deployment.DeployMonitoringConsole(ctx, name, licenseManagerRef) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, name, mc) + return mc +} + +// VerifyIndexerCPULimits verifies CPU limits on all indexer pods in a single-site cluster +func (testcaseenv *TestCaseEnv) VerifyIndexerCPULimits(deployment *Deployment, deploymentName string, indexerCount int, expectedCPULimit string) { + for i := 0; i < indexerCount; i++ { + podName := fmt.Sprintf(IndexerPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifySearchHeadCPULimits verifies CPU limits on all search head pods +func (testcaseenv *TestCaseEnv) VerifySearchHeadCPULimits(deployment *Deployment, deploymentName string, searchHeadCount int, expectedCPULimit string) { + for i := 0; i < searchHeadCount; i++ { + podName := fmt.Sprintf(SearchHeadPod, deploymentName, i) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// VerifyM4ComponentsReady verifies multisite indexers, multisite status, and SHC are ready (without CM check or RFSF). +func (testcaseenv *TestCaseEnv) VerifyM4ComponentsReady(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyIndexersReady(ctx, deployment, siteCount) + testcaseenv.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount) + testcaseenv.VerifySearchHeadClusterReady(ctx, deployment) +} + +// VerifyMCVersionChangedAndReady waits for the MC resource version to change then verifies MC is ready. +func (testcaseenv *TestCaseEnv) VerifyMCVersionChangedAndReady(ctx context.Context, deployment *Deployment, mc *enterpriseApi.MonitoringConsole, resourceVersion string) { + testcaseenv.VerifyCustomResourceVersionChanged(ctx, deployment, mc, resourceVersion) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc) +} + +// VerifyClusterReadyAndRFSF is a common verification pattern that checks cluster is ready and RF/SF is met +func (testcaseenv *TestCaseEnv) VerifyClusterReadyAndRFSF(ctx context.Context, deployment *Deployment) { + testcaseenv.VerifyC3ClusterReady(ctx, deployment, testcaseenv.VerifyClusterManagerReady) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// VerifyMultisiteClusterReadyAndRFSF is a common verification pattern for multisite clusters +func (testcaseenv *TestCaseEnv) VerifyMultisiteClusterReadyAndRFSF(ctx context.Context, deployment *Deployment, siteCount int) { + testcaseenv.VerifyClusterManagerReady(ctx, deployment) + testcaseenv.VerifyM4ComponentsReady(ctx, deployment, siteCount) + testcaseenv.VerifyRFSFMet(ctx, deployment) +} + +// TriggerAndVerifyTelemetry is a common pattern for telemetry verification +func (testcaseenv *TestCaseEnv) TriggerAndVerifyTelemetry(ctx context.Context, deployment *Deployment, prevSubmissionTime string) { + testcaseenv.TriggerTelemetrySubmission(ctx, deployment) + testcaseenv.VerifyTelemetry(ctx, deployment, prevSubmissionTime) +} + +// VerifyProbeConfigAndScripts verifies probe config map exists and probe scripts are present on all pods. +// If includeStartup is true, the startup probe script is also checked. +func (testcaseenv *TestCaseEnv) VerifyProbeConfigAndScripts(ctx context.Context, deployment *Deployment, includeStartup bool) { + testcaseenv.Log.Info("Get config map for livenessProbe and readinessProbe") + configMapName := enterprise.GetProbeConfigMapName(testcaseenv.GetName()) + _, err := GetConfigMap(ctx, deployment, testcaseenv.GetName(), configMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", configMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName()} + if includeStartup { + scriptsNames = append(scriptsNames, enterprise.GetStartupScriptName()) + } + allPods := DumpGetPods(testcaseenv.GetName()) + testcaseenv.VerifyFilesInDirectoryOnPod(ctx, deployment, allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) +} + +// NewStandaloneSpecWithMCRef creates a StandaloneSpec with a MonitoringConsoleRef set to the given MC name. +func NewStandaloneSpecWithMCRef(image string, mcName string) enterpriseApi.StandaloneSpec { + return enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: image, + }, + Volumes: []corev1.Volume{}, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: mcName, + }, + }, + } +} + +// VerifyLMConfiguredOnMC verifies that the License Manager is configured on the Monitoring Console pod. +func VerifyLMConfiguredOnMC(ctx context.Context, deployment *Deployment) { + monitoringConsolePodName := fmt.Sprintf(MonitoringConsolePod, deployment.GetName()) + VerifyLMConfiguredOnPod(ctx, deployment, monitoringConsolePodName) +} + +// StandardC3Verification performs the standard set of verifications for a C3 cluster +// This includes cluster ready, RF/SF met, and monitoring console ready +func (testcaseenv *TestCaseEnv) StandardC3Verification(ctx context.Context, deployment *Deployment, mcName string, mc *enterpriseApi.MonitoringConsole) { + testcaseenv.VerifyClusterReadyAndRFSF(ctx, deployment) + testcaseenv.VerifyMonitoringConsoleReady(ctx, deployment, mcName, mc) +} diff --git a/test/testenv/cr_update_helpers.go b/test/testenv/cr_update_helpers.go new file mode 100644 index 000000000..492b26187 --- /dev/null +++ b/test/testenv/cr_update_helpers.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetInstanceWithExpect is a wrapper around GetInstance that includes Gomega expectations +func GetInstanceWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, instanceName string, errorMsg string) { + err := deployment.GetInstance(ctx, instanceName, obj) + Expect(err).To(Succeed(), errorMsg, "Instance Name", instanceName) +} + +// UpdateCRWithExpect is a wrapper around UpdateCR that includes Gomega expectations +func UpdateCRWithExpect(ctx context.Context, deployment *Deployment, obj client.Object, errorMsg string) { + err := deployment.UpdateCR(ctx, obj) + Expect(err).To(Succeed(), errorMsg) +} diff --git a/test/testenv/deployment.go b/test/testenv/deployment.go index 395722983..17059fb18 100644 --- a/test/testenv/deployment.go +++ b/test/testenv/deployment.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,7 +154,7 @@ func (d *Deployment) DeployStandalone(ctx context.Context, name string, mcRef st // If license file specified, deploy License Manager if LicenseManager != "" && d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -326,7 +326,7 @@ func (d *Deployment) OperatorPodExecCommand(ctx context.Context, podName string, return stdout.String(), stderr.String(), nil } -// DeployLicenseManager deploys the license manager instance +// DeployLicenseManager deploys the License Manager instance func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*enterpriseApi.LicenseManager, error) { if d.testenv.licenseFilePath == "" { @@ -344,7 +344,7 @@ func (d *Deployment) DeployLicenseManager(ctx context.Context, name string) (*en return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMaster deploys the license manager instance +// DeployLicenseMaster deploys the License Master instance func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*enterpriseApiV3.LicenseMaster, error) { if d.testenv.licenseFilePath == "" { @@ -362,9 +362,9 @@ func (d *Deployment) DeployLicenseMaster(ctx context.Context, name string) (*ent return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeployClusterManager deploys the cluster manager +// DeployClusterManager deploys the Cluster Manager func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterManager(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -382,9 +382,9 @@ func (d *Deployment) DeployClusterManager(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMaster deploys the cluster manager +// DeployClusterMaster deploys the Cluster Master func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name, "LicenseRef", LicenseManagerName) + d.testenv.Log.Info("Deploying Cluster Master", "name", name, "LicenseRef", LicenseManagerName) cm := newClusterMaster(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { cm.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -402,9 +402,9 @@ func (d *Deployment) DeployClusterMaster(ctx context.Context, name, LicenseManag return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployClusterManagerWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterManagerWithSmartStoreIndexes deploys the Cluster Manager with SmartStore indexes func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName string, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -416,9 +416,9 @@ func (d *Deployment) DeployClusterManagerWithSmartStoreIndexes(ctx context.Conte return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithSmartStoreIndexes deploys the cluster manager with smartstore indexes +// DeployClusterMasterWithSmartStoreIndexes deploys the Cluster Master with SmartStore indexes func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Context, name, LicenseManagerName, ansibleConfig string, smartstorespec enterpriseApi.SmartStoreSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterMasterWithGivenIndexes(name, d.testenv.namespace, LicenseManagerName, ansibleConfig, d.testenv.splunkImage, smartstorespec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -430,12 +430,12 @@ func (d *Deployment) DeployClusterMasterWithSmartStoreIndexes(ctx context.Contex return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeployIndexerCluster deploys the indexer cluster +// DeployIndexerCluster deploys the Indexer Cluster func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseManagerName string, count int, clusterManagerRef string, ansibleConfig string, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IndexerCluster, error) { - d.testenv.Log.Info("Deploying indexer cluster", "name", name, "CM", clusterManagerRef) + d.testenv.Log.Info("Deploying Indexer Cluster", "name", name, "CM", clusterManagerRef) indexer := newIndexerCluster(name, d.testenv.namespace, LicenseManagerName, count, clusterManagerRef, ansibleConfig, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(indexer) - d.testenv.Log.Info("indexer cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Indexer Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, indexer) if err != nil { return nil, err @@ -444,14 +444,14 @@ func (d *Deployment) DeployIndexerCluster(ctx context.Context, name, LicenseMana return deployed.(*enterpriseApi.IndexerCluster), err } -// DeployIngestorCluster deploys the ingestor cluster +// DeployIngestorCluster deploys the Ingestor Cluster func (d *Deployment) DeployIngestorCluster(ctx context.Context, name string, count int, queue, os corev1.ObjectReference, serviceAccountName string) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster", "name", name) + d.testenv.Log.Info("Deploying Ingestor Cluster", "name", name) ingestor := newIngestorCluster(name, d.testenv.namespace, count, d.testenv.splunkImage, queue, os, serviceAccountName) pdata, _ := json.Marshal(ingestor) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, name, ingestor) if err != nil { return nil, err @@ -492,13 +492,13 @@ func (d *Deployment) DeployObjectStorage(ctx context.Context, name string, objSt return deployed.(*enterpriseApi.ObjectStorage), err } -// DeployIngestorClusterWithAdditionalConfiguration deploys the ingestor cluster with additional configuration +// DeployIngestorClusterWithAdditionalConfiguration deploys the Ingestor Cluster with additional configuration func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx context.Context, ic *enterpriseApi.IngestorCluster) (*enterpriseApi.IngestorCluster, error) { - d.testenv.Log.Info("Deploying ingestor cluster with additional configuration", "name", ic.Name) + d.testenv.Log.Info("Deploying Ingestor Cluster with additional configuration", "name", ic.Name) pdata, _ := json.Marshal(ic) - d.testenv.Log.Info("ingestor cluster spec", "cr", string(pdata)) + d.testenv.Log.Info("Ingestor Cluster spec", "cr", string(pdata)) deployed, err := d.deployCR(ctx, ic.Name, ic) if err != nil { return nil, err @@ -507,9 +507,9 @@ func (d *Deployment) DeployIngestorClusterWithAdditionalConfiguration(ctx contex return deployed.(*enterpriseApi.IngestorCluster), err } -// DeploySearchHeadCluster deploys a search head cluster +// DeploySearchHeadCluster deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadCluster(ctx context.Context, name, ClusterManagerRef, LicenseManagerName string, ansibleConfig string, mcRef string) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) sh := newSearchHeadCluster(name, d.testenv.namespace, ClusterManagerRef, LicenseManagerName, ansibleConfig, d.testenv.splunkImage) if mcRef != "" { sh.Spec.MonitoringConsoleRef = corev1.ObjectReference{ @@ -553,7 +553,7 @@ func (d *Deployment) deployCR(ctx context.Context, name string, cr client.Object return nil }) - // Returns once we can retrieve the lm instance + // Returns once we can retrieve the LM instance if err := wait.PollImmediate(PollInterval, DefaultTimeout, func() (bool, error) { key := client.ObjectKey{Name: name, Namespace: d.testenv.namespace} err := d.testenv.GetKubeClient().Get(ctx, key, cr) @@ -725,7 +725,7 @@ func (d *Deployment) DeleteCR(ctx context.Context, cr client.Object) error { return err } -// DeploySingleSiteCluster deploys a lm and indexer cluster (shc optional) +// DeploySingleSiteCluster deploys a LM and Indexer Cluster (SHC optional) func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, indexerReplicas int, shc bool, mcRef string) error { var LicenseManager string @@ -734,13 +734,13 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -751,26 +751,26 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i } if strings.Contains(name, "master") { - // Deploy the cluster master + // Deploy the Cluster Master _, err := d.DeployClusterMaster(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } else { - // Deploy the cluster manager + // Deploy the Cluster Manager _, err := d.DeployClusterManager(ctx, name, LicenseManager, "", mcRef) if err != nil { return err } } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err := d.DeployIndexerCluster(ctx, name+"-idxc", LicenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err } - // Deploy the SH cluster + // Deploy the SH Cluster if shc { _, err = d.DeploySearchHeadCluster(ctx, name+"-shc", name, LicenseManager, "", mcRef) if err != nil { @@ -781,7 +781,7 @@ func (d *Deployment) DeploySingleSiteCluster(ctx context.Context, name string, i return nil } -// DeployMultisiteClusterMasterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseMaster string @@ -796,7 +796,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont LicenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -843,7 +843,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHead(ctx context.Cont return nil } -// DeployMultisiteClusterWithSearchHead deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHead deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string @@ -852,13 +852,13 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n if d.testenv.licenseFilePath != "" { // Enable LM to be tested if strings.Contains(name, "master") { - // Deploy the license master + // Deploy the License Master _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err } } else { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -868,7 +868,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -915,14 +915,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHead(ctx context.Context, n return nil } -// DeployMultisiteCluster deploys a lm, cluster-manager, indexers in multiple sites +// DeployMultisiteCluster deploys a LM, Cluster Manager, and indexers in multiple sites func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, indexerReplicas int, siteCount int, mcRef string) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -931,7 +931,7 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -965,13 +965,13 @@ func (d *Deployment) DeployMultisiteCluster(ctx context.Context, name string, in return nil } -// DeployStandaloneWithLMaster deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLMaster deploys a Standalone Splunk Enterprise instance with License Master on the specified testenv func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return nil, err @@ -992,13 +992,13 @@ func (d *Deployment) DeployStandaloneWithLMaster(ctx context.Context, name strin return deployed.(*enterpriseApi.Standalone), err } -// DeployStandaloneWithLM deploys a standalone splunk enterprise instance with license manager on the specified testenv +// DeployStandaloneWithLM deploys a Standalone Splunk Enterprise instance with License Manager on the specified testenv func (d *Deployment) DeployStandaloneWithLM(ctx context.Context, name string, mcRef string) (*enterpriseApi.Standalone, error) { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return nil, err @@ -1051,14 +1051,14 @@ func (d *Deployment) DeployStandaloneWithGivenSmartStoreSpec(ctx context.Context return deployed.(*enterpriseApi.Standalone), err } -// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a lm, cluster-manager, indexers in multiple sites and SH clusters +// DeployMultisiteClusterWithSearchHeadAndIndexes deploys a LM, Cluster Manager, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1067,7 +1067,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1106,14 +1106,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndIndexes(ctx context. return err } -// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a lm, cluster-master, indexers in multiple sites and SH clusters +// DeployMultisiteClusterMasterWithSearchHeadAndIndexes deploys a LM, Cluster Master, indexers in multiple sites and SH Clusters func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx context.Context, name string, indexerReplicas int, siteCount int, indexesSecret string, smartStoreSpec enterpriseApi.SmartStoreSpec) error { var LicenseManager string // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1122,7 +1122,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co LicenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1161,9 +1161,9 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndIndexes(ctx co return err } -// DeployClusterManagerWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterManagerWithGivenSpec deploys the Cluster Manager with given SPEC func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.ClusterManagerSpec) (*enterpriseApi.ClusterManager, error) { - d.testenv.Log.Info("Deploying cluster-manager", "name", name) + d.testenv.Log.Info("Deploying Cluster Manager", "name", name) cm := newClusterManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1172,9 +1172,9 @@ func (d *Deployment) DeployClusterManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.ClusterManager), err } -// DeployClusterMasterWithGivenSpec deploys the cluster manager with given SPEC +// DeployClusterMasterWithGivenSpec deploys the Cluster Master with given SPEC func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.ClusterMasterSpec) (*enterpriseApiV3.ClusterMaster, error) { - d.testenv.Log.Info("Deploying cluster-master", "name", name) + d.testenv.Log.Info("Deploying Cluster Master", "name", name) cm := newClusterMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, cm) if err != nil { @@ -1183,17 +1183,17 @@ func (d *Deployment) DeployClusterMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.ClusterMaster), err } -// DeploySearchHeadClusterWithGivenSpec deploys a search head cluster +// DeploySearchHeadClusterWithGivenSpec deploys a Search Head Cluster func (d *Deployment) DeploySearchHeadClusterWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.SearchHeadClusterSpec) (*enterpriseApi.SearchHeadCluster, error) { - d.testenv.Log.Info("Deploying search head cluster", "name", name) + d.testenv.Log.Info("Deploying Search Head Cluster", "name", name) indexer := newSearchHeadClusterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, indexer) return deployed.(*enterpriseApi.SearchHeadCluster), err } -// DeployLicenseManagerWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseManagerWithGivenSpec deploys the License Manager with given SPEC func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name string, spec enterpriseApi.LicenseManagerSpec) (*enterpriseApi.LicenseManager, error) { - d.testenv.Log.Info("Deploying license-manager", "name", name) + d.testenv.Log.Info("Deploying License Manager", "name", name) lm := newLicenseManagerWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1202,9 +1202,9 @@ func (d *Deployment) DeployLicenseManagerWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApi.LicenseManager), err } -// DeployLicenseMasterWithGivenSpec deploys the license manager with given SPEC +// DeployLicenseMasterWithGivenSpec deploys the License Master with given SPEC func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name string, spec enterpriseApiV3.LicenseMasterSpec) (*enterpriseApiV3.LicenseMaster, error) { - d.testenv.Log.Info("Deploying license-master", "name", name) + d.testenv.Log.Info("Deploying License Master", "name", name) lm := newLicenseMasterWithGivenSpec(name, d.testenv.namespace, spec) deployed, err := d.deployCR(ctx, name, lm) if err != nil { @@ -1213,7 +1213,7 @@ func (d *Deployment) DeployLicenseMasterWithGivenSpec(ctx context.Context, name return deployed.(*enterpriseApiV3.LicenseMaster), err } -// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseManager string) (*enterpriseApi.ClusterManager, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApi.ClusterManager{} @@ -1222,14 +1222,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1251,7 +1251,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1279,7 +1279,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1291,7 +1291,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx contex return cm, idxc, sh, nil } -// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys indexer cluster (lm, shc optional) with app framework spec +// DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec deploys Indexer Cluster (LM, SHC optional) with app framework spec func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx context.Context, name string, indexerReplicas int, shc bool, appFrameworkSpecIdxc enterpriseApi.AppFrameworkSpec, appFrameworkSpecShc enterpriseApi.AppFrameworkSpec, mcName string, licenseMaster string) (*enterpriseApiV3.ClusterMaster, *enterpriseApi.IndexerCluster, *enterpriseApi.SearchHeadCluster, error) { cm := &enterpriseApiV3.ClusterMaster{} @@ -1300,14 +1300,14 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1329,7 +1329,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx return cm, idxc, sh, err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster idxc, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return cm, idxc, sh, err @@ -1357,7 +1357,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx } pdata, _ := json.Marshal(shSpec) - d.testenv.Log.Info("Search head Spec", "cr", pdata) + d.testenv.Log.Info("Search Head Spec", "cr", pdata) if shc { sh, err = d.DeploySearchHeadClusterWithGivenSpec(ctx, name+"-shc", shSpec) @@ -1378,14 +1378,14 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, licenseManager) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1440,7 +1440,7 @@ func (d *Deployment) DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1482,14 +1482,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, licenseMaster) if err != nil { return cm, idxc, sh, err } } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1544,7 +1544,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1577,14 +1577,14 @@ func (d *Deployment) DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(c return cm, idxc, sh, nil } -// DeploySingleSiteClusterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseManager := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1593,7 +1593,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte licenseManager = name } - // Deploy the cluster manager + // Deploy the Cluster Manager cmSpec := enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1614,7 +1614,7 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseManager, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1649,14 +1649,14 @@ func (d *Deployment) DeploySingleSiteClusterWithGivenMonitoringConsole(ctx conte return nil } -// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys indexer cluster (lm, shc optional) with given monitoring console +// DeploySingleSiteClusterMasterWithGivenMonitoringConsole deploys Indexer Cluster (LM, SHC optional) with given Monitoring Console func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx context.Context, name string, indexerReplicas int, shc bool, monitoringConsoleName string) error { licenseMaster := "" // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1665,7 +1665,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx licenseMaster = name } - // Deploy the cluster manager + // Deploy the Cluster Master cmSpec := enterpriseApiV3.ClusterMasterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1686,7 +1686,7 @@ func (d *Deployment) DeploySingleSiteClusterMasterWithGivenMonitoringConsole(ctx return err } - // Deploy the indexer cluster + // Deploy the Indexer Cluster _, err = d.DeployIndexerCluster(ctx, name+"-idxc", licenseMaster, indexerReplicas, name, "", corev1.ObjectReference{}, corev1.ObjectReference{}, "") if err != nil { return err @@ -1728,7 +1728,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseManager(ctx, name) if err != nil { return err @@ -1737,7 +1737,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con licenseManager = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1791,7 +1791,7 @@ func (d *Deployment) DeployMultisiteClusterWithMonitoringConsole(ctx context.Con multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-manager") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ @@ -1828,7 +1828,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte // If license file specified, deploy License Manager if d.testenv.licenseFilePath != "" { - // Deploy the license manager + // Deploy the License Manager _, err := d.DeployLicenseMaster(ctx, name) if err != nil { return err @@ -1837,7 +1837,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte licenseMaster = name } - // Deploy the cluster-manager + // Deploy the Cluster Manager defaults := `splunk: multisite_master: localhost all_sites: site1,site2,site3 @@ -1891,7 +1891,7 @@ func (d *Deployment) DeployMultisiteClusterMasterWithMonitoringConsole(ctx conte multisite_master: splunk-%s-%s-service site: site0 `, name, "cluster-master") - // Deploy the SH cluster + // Deploy the SH Cluster shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ diff --git a/test/testenv/gcputils.go b/test/testenv/gcputils.go index 78c41d1aa..e6b9a0b1e 100644 --- a/test/testenv/gcputils.go +++ b/test/testenv/gcputils.go @@ -120,9 +120,9 @@ func CheckPrefixExistsOnGCP(prefix string) bool { logf.Log.Error(err, "Error listing objects in GCP bucket") return false } - logf.Log.Info("CHECKING OBJECT", "OBJECT", objAttrs.Name) + logf.Log.Info("CHECKING OBJECT", "object", objAttrs.Name) if strings.Contains(objAttrs.Name, prefix) { - logf.Log.Info("Prefix found in bucket", "Prefix", prefix, "Object", objAttrs.Name) + logf.Log.Info("Prefix found in bucket", "prefix", prefix, "object", objAttrs.Name) return true } } @@ -146,12 +146,12 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create the bucket err = client.Client.Bucket(bucketName).Create(ctx, gcpProjectID, nil) if err != nil { - logf.Log.Error(err, "Failed to create bucket", "Bucket Name", bucketName) + logf.Log.Error(err, "Failed to create bucket", "bucketName", bucketName) return err } - logf.Log.Info("Bucket created", "Bucket Name", bucketName) + logf.Log.Info("Bucket created", "bucketName", bucketName) } else if err != nil { - logf.Log.Error(err, "Error checking bucket attributes", "Bucket Name", bucketName) + logf.Log.Error(err, "Error checking bucket attributes", "bucketName", bucketName) return err } @@ -161,16 +161,16 @@ func CreateBucketAndPathIfNotExist(bucketName, path string) error { // Create a zero-length object to represent the path wc := client.Client.Bucket(bucketName).Object(path).NewWriter(ctx) if _, err := wc.Write([]byte{}); err != nil { - logf.Log.Error(err, "Failed to create path", "Path", path) + logf.Log.Error(err, "Failed to create path", "path", path) return err } if err := wc.Close(); err != nil { - logf.Log.Error(err, "Failed to finalize path creation", "Path", path) + logf.Log.Error(err, "Failed to finalize path creation", "path", path) return err } - logf.Log.Info("Path created", "Path", path) + logf.Log.Info("Path created", "path", path) } else if err != nil { - logf.Log.Error(err, "Error checking path attributes", "Path", path) + logf.Log.Error(err, "Error checking path attributes", "path", path) return err } @@ -209,7 +209,7 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string objectPath := filepath.Join(gcpFilePath, objectName) rc, err := client.Client.Bucket(bucketName).Object(objectPath).NewReader(ctx) if err != nil { - logf.Log.Error(err, "Failed to create reader for object", "Object", objectName) + logf.Log.Error(err, "Failed to create reader for object", "object", objectName) return "", err } defer rc.Close() @@ -217,14 +217,14 @@ func DownloadFileFromGCP(bucketName, objectName, gcpFilePath, downloadDir string localPath := filepath.Join(downloadDir, objectName) file, err := os.Create(localPath) if err != nil { - logf.Log.Error(err, "Failed to create local file", "Filename", localPath) + logf.Log.Error(err, "Failed to create local file", "filename", localPath) return "", err } defer file.Close() written, err := io.Copy(file, rc) if err != nil { - logf.Log.Error(err, "Failed to download object", "Object", objectName) + logf.Log.Error(err, "Failed to download object", "object", objectName) return "", err } @@ -319,18 +319,18 @@ func DeleteFileOnGCP(bucketName, objectName string) error { err = client.Client.Bucket(bucketName).Object(objectName).Delete(ctx) if err != nil && err != storage.ErrObjectNotExist { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", objectName, "bucketName", bucketName) return err } // Optionally, verify deletion _, err = client.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) if err == storage.ErrObjectNotExist { - logf.Log.Info("Deleted file on GCP", "File Name", objectName, "Bucket", bucketName) + logf.Log.Info("Deleted file on GCP", "fileName", objectName, "bucket", bucketName) return nil } if err != nil { - logf.Log.Error(err, "Error verifying deletion of object", "Object Name", objectName, "Bucket Name", bucketName) + logf.Log.Error(err, "Error verifying deletion of object", "objectName", objectName, "bucketName", bucketName) return err } @@ -342,12 +342,12 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { resp := GetFileListOnGCP(bucketName, path) var files []string for _, obj := range resp { - logf.Log.Info("CHECKING OBJECT", "OBJECT", obj.Name) + logf.Log.Info("CHECKING OBJECT", "object", obj.Name) if strings.HasPrefix(obj.Name, path) { filename := strings.TrimPrefix(obj.Name, path) // This condition filters out directories as GCP returns objects with their full paths if len(filename) > 1 && !strings.HasSuffix(filename, "/") { - logf.Log.Info("File found in bucket", "Path", path, "Object", obj.Name) + logf.Log.Info("File found in bucket", "path", path, "object", obj.Name) files = append(files, filename) } } @@ -358,10 +358,10 @@ func GetFilesInPathOnGCP(bucketName, path string) []string { // DownloadFilesFromGCP downloads a list of files from a GCP bucket to a local directory func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from GCP", "File name", key) + logf.Log.Info("Downloading file from GCP", "fileName", key) _, err := DownloadFileFromGCP(bucketName, key, gcpAppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -372,22 +372,22 @@ func DownloadFilesFromGCP(bucketName, gcpAppDir, downloadDir string, appList []s func UploadFilesToGCP(bucketName, gcpTestDir string, appList []string, uploadDir string) ([]string, error) { var uploadedFiles []string for _, key := range appList { - logf.Log.Info("Uploading file to GCP", "File name", key) - logf.Log.Info("Using bucket", "Bucket", bucketName, "Path", gcpTestDir, "Upload Dir", uploadDir) + logf.Log.Info("Uploading file to GCP", "fileName", key) + logf.Log.Info("Using bucket", "bucket", bucketName, "path", gcpTestDir, "uploadDir", uploadDir) fileLocation := filepath.Join(uploadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } defer fileBody.Close() objectPath, err := UploadFileToGCP(bucketName, key, gcpTestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File uploaded to GCP", "File name", objectPath) + logf.Log.Info("File uploaded to GCP", "fileName", objectPath) uploadedFiles = append(uploadedFiles, objectPath) } return uploadedFiles, nil diff --git a/test/testenv/ingest_utils.go b/test/testenv/ingest_utils.go index 2c0403b1e..1f28693bf 100644 --- a/test/testenv/ingest_utils.go +++ b/test/testenv/ingest_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test/testenv/lmutil.go b/test/testenv/lmutil.go index 9f7aa5dd7..e58cf2b2d 100644 --- a/test/testenv/lmutil.go +++ b/test/testenv/lmutil.go @@ -1,18 +1,16 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package testenv import ( @@ -56,6 +54,6 @@ func CheckLicenseManagerConfigured(ctx context.Context, deployment *Deployment, return false } licenseManager := restResponse.Entry[0].Content.ManagerURI - logf.Log.Info("License Manager configuration on POD", "POD", podName, "License Manager", licenseManager) + logf.Log.Info("License Manager configuration on POD", "pod", podName, "licenseManager", licenseManager) return strings.Contains(licenseManager, "license-manager-service:8089") || strings.Contains(licenseManager, "license-master-service:8089") } diff --git a/test/testenv/mcutil.go b/test/testenv/mcutil.go index 36250ae96..0cd9448ee 100644 --- a/test/testenv/mcutil.go +++ b/test/testenv/mcutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,7 +60,7 @@ func CheckMCPodReady(ns string) bool { return false } stsSlice := strings.Fields(stsLine) - logf.Log.Info("MC statefulset found", "POD", stsSlice[0], "READY", stsSlice[1]) + logf.Log.Info("MC statefulset found", "pod", stsSlice[0], "ready", stsSlice[1]) stsReady := strings.Contains(stsSlice[1], "1/1") // Check Status of monitoring console pod @@ -69,13 +69,13 @@ func CheckMCPodReady(ns string) bool { return false } podSlice := strings.Fields(podLine) - logf.Log.Info("MC Pod Found", "POD", podSlice[0], "READY", podSlice[1]) + logf.Log.Info("MC Pod Found", "pod", podSlice[0], "ready", podSlice[1]) podReady := strings.Contains(podSlice[1], "1/1") && strings.Contains(podSlice[2], "Running") return stsReady && podReady } -// GetConfiguredPeers get list of Peers Configured on Montioring Console +// GetConfiguredPeers get list of Peers Configured on Monitoring Console func GetConfiguredPeers(ns string, mcName string) []string { podName := fmt.Sprintf(MonitoringConsolePod, mcName) var peerList []string @@ -100,22 +100,10 @@ func GetConfiguredPeers(ns string, mcName string) []string { } } } - logf.Log.Info("Peer List found on MC Pod", "MC POD", podName, "Configured Peers", peerList) + logf.Log.Info("Peer List found on MC Pod", "mcPod", podName, "configuredPeers", peerList) return peerList } -// DeleteMCPod delete monitoring console deployment -func DeleteMCPod(ns string) { - mcSts := fmt.Sprintf(MonitoringConsoleSts, ns) - output, err := exec.Command("kubectl", "delete", "sts", "-n", ns, mcSts).Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete sts -n %s %s", ns, mcSts) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("Monitoring Console Stateful Set deleted", "Statefulset", mcSts, "stdout", output) - } -} - // CheckPodNameOnMC Check given pod is configured on Monitoring console pod func CheckPodNameOnMC(ns string, mcName string, podName string) bool { // Get Peers configured on Monitoring Console @@ -124,7 +112,7 @@ func CheckPodNameOnMC(ns string, mcName string, podName string) bool { found := false for _, peer := range peerList { if strings.Contains(peer, podName) { - logf.Log.Info("Check Peer matches on pod", "Pod Name", podName, "Peer in peer list", peer) + logf.Log.Info("Check Peer matches on pod", "podName", podName, "peerInPeerList", peer) found = true break } @@ -157,12 +145,12 @@ func GetMCConfigMap(ctx context.Context, deployment *Deployment, ns string, mcNa logf.Log.Error(err, "Failed to get Monitoring Console Config Map") return mcConfigMap, err } - logf.Log.Info("MC Config Map contents", "MC CONFIG MAP NAME", mcConfigMapName, "Data", mcConfigMap.Data) + logf.Log.Info("MC Config Map contents", "mcConfigMapName", mcConfigMapName, "data", mcConfigMap.Data) return mcConfigMap, err } // CheckPodNameInString checks for pod name in string func CheckPodNameInString(podName string, configString string) bool { - logf.Log.Info("Check MC Config String has Pod configured", "Monitoring Console Config Map Pod Config String", configString, "POD String", podName) + logf.Log.Info("Check MC Config String has Pod configured", "configString", configString, "podName", podName) return strings.Contains(configString, podName) } diff --git a/test/testenv/multisite_helpers.go b/test/testenv/multisite_helpers.go new file mode 100644 index 000000000..dd837ffe8 --- /dev/null +++ b/test/testenv/multisite_helpers.go @@ -0,0 +1,78 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" +) + +// VerifyIndexOnAllSites verifies that an index exists on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexFoundOnPod(ctx, deployment, podName, indexName) + } +} + +// IngestDataOnAllSites ingests data to an index on all indexer pods across all sites +func IngestDataOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + logFile := fmt.Sprintf("test-log-%s.log", RandomDNSName(3)) + CreateMockLogfile(logFile, LogLineCount) + IngestFileViaMonitor(ctx, logFile, indexName, podName, deployment) + } +} + +// RollHotToWarmOnAllSites rolls hot buckets to warm on all indexer pods across all sites +func RollHotToWarmOnAllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + RollHotToWarm(ctx, deployment, podName, indexName) + } +} + +// VerifyIndexOnS3AllSites verifies that an index exists on S3 for all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyIndexOnS3AllSites(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyIndexExistsOnS3(ctx, deployment, indexName, podName) + } +} + +// VerifyCPULimitsOnAllSites verifies CPU limits on all indexer pods across all sites +func (testcaseenv *TestCaseEnv) VerifyCPULimitsOnAllSites(deployment *Deployment, deploymentName string, siteCount int, expectedCPULimit string) { + for siteNumber := 1; siteNumber <= siteCount; siteNumber++ { + podName := fmt.Sprintf(MultiSiteIndexerPod, deploymentName, siteNumber, 0) + testcaseenv.VerifyCPULimits(deployment, podName, expectedCPULimit) + } +} + +// MultisiteIndexerWorkflow encapsulates the common workflow for multisite indexer operations: +// verify index, ingest data, roll to warm, verify on S3 +func (testcaseenv *TestCaseEnv) MultisiteIndexerWorkflow(ctx context.Context, deployment *Deployment, deploymentName string, siteCount int, indexName string) { + // Verify index exists on all sites + testcaseenv.VerifyIndexOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Ingest data on all sites + IngestDataOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Roll hot to warm on all sites + RollHotToWarmOnAllSites(ctx, deployment, deploymentName, siteCount, indexName) + + // Verify index on S3 for all sites + testcaseenv.VerifyIndexOnS3AllSites(ctx, deployment, deploymentName, siteCount, indexName) +} diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index 551a377d0..e093e5008 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -60,19 +60,6 @@ func GetIndexOnPod(ctx context.Context, deployment *Deployment, podName string, return indexFound, indexData } -// RestartSplunk Restart splunk inside the container -func RestartSplunk(ctx context.Context, deployment *Deployment, podName string) bool { - stdin := "/opt/splunk/bin/splunk restart -auth admin:$(cat /mnt/splunk-secrets/password)" - command := []string{"/bin/sh"} - stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) - if err != nil { - logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) - return false - } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - return true -} - // RollHotToWarm rolls hot buckets to warm for a given index and pod func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, indexName string) bool { stdin := "/opt/splunk/bin/splunk _internal call /data/indexes/" + indexName + "/roll-hot-buckets admin:$(cat /mnt/splunk-secrets/password)" diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 445e395f1..3921844e2 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -32,11 +32,6 @@ var ( s3deleteWaitTime = 60 ) -// GetSmartStoreIndexesBucet returns smartstore test bucket name -func GetSmartStoreIndexesBucet() string { - return testIndexesS3Bucket -} - // GetDefaultS3Region returns default AWS Region func GetDefaultS3Region() string { return s3Region @@ -53,9 +48,9 @@ func CheckPrefixExistsOnS3(prefix string) bool { resp := GetFileListOnS3(dataBucket, prefix) for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, prefix) { - logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key) + logf.Log.Info("Prefix found on bucket", "prefix", prefix, "key", *key.Key) return true } } @@ -94,7 +89,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d // Create empty file on OS File System file, err := os.Create(filepath.Join(downloadDir, filename)) if err != nil { - logf.Log.Error(err, "Failed to create file", "Filename", file) + logf.Log.Error(err, "Failed to create file", "filename", file) } defer file.Close() @@ -112,7 +107,7 @@ func DownloadFileFromS3(dataBucket string, filename string, s3FilePath string, d }) if err != nil { - logf.Log.Error(err, "Failed to download file", "Bucket", dataBucket, "Path", s3FilePath, "Filename", filename) + logf.Log.Error(err, "Failed to download file", "bucket", dataBucket, "path", s3FilePath, "filename", filename) return "", err } @@ -174,7 +169,7 @@ func DeleteFileOnS3(bucket string, filename string) error { svc := s3.NewFromConfig(*cfg) _, err = svc.DeleteObject(context.TODO(), &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(filename)}) if err != nil { - logf.Log.Error(err, "Unable to delete object from bucket", "Object Name", filename, "Bucket Name", bucket) + logf.Log.Error(err, "Unable to delete object from bucket", "objectName", filename, "bucketName", bucket) } waiter := s3.NewObjectNotExistsWaiter(svc) @@ -182,7 +177,7 @@ func DeleteFileOnS3(bucket string, filename string) error { Bucket: aws.String(bucket), Key: aws.String(filename), }, time.Duration(time.Duration(s3deleteWaitTime).Seconds())) - logf.Log.Info("Deleted file on S3", "File Name", filename, "Bucket", bucket) + logf.Log.Info("Deleted file on S3", "fileName", filename, "bucket", bucket) return err } @@ -191,12 +186,12 @@ func GetFilesInPathOnS3(bucket string, path string) []string { resp := GetFileListOnS3(bucket, path) var files []string for _, key := range resp { - logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) + logf.Log.Info("CHECKING KEY ", "key", *key.Key) if strings.Contains(*key.Key, path) { filename := strings.Replace(*key.Key, path, "", -1) // This condition filters out directories as GetFileListOnS3 returns files and directories with their absolute path's if len(filename) > 1 { - logf.Log.Info("File found on bucket", "Prefix", path, "KEY", *key.Key) + logf.Log.Info("File found on bucket", "prefix", path, "key", *key.Key) files = append(files, filename) } } @@ -207,10 +202,10 @@ func GetFilesInPathOnS3(bucket string, path string) []string { // DownloadFilesFromS3 download given list of files from S3 to the given directory func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading file from S3", "File name", key) + logf.Log.Info("Downloading file from S3", "fileName", key) _, err := DownloadFileFromS3(testDataS3Bucket, key, s3AppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to download file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "fileName", key) return err } } @@ -221,19 +216,19 @@ func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir s func UploadFilesToS3(testS3Bucket string, s3TestDir string, applist []string, downloadDir string) ([]string, error) { var uploadedFiles []string for _, key := range applist { - logf.Log.Info("Uploading file to S3", "File name", key) + logf.Log.Info("Uploading file to S3", "fileName", key) fileLocation := filepath.Join(downloadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open file", "File name", key) + logf.Log.Error(err, "Unable to open file", "fileName", key) return nil, err } fileName, err := UploadFileToS3(testS3Bucket, key, s3TestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File name", key) + logf.Log.Error(err, "Unable to upload file", "fileName", key) return nil, err } - logf.Log.Info("File upload to test S3", "File name", fileName) + logf.Log.Info("File upload to test S3", "fileName", fileName) uploadedFiles = append(uploadedFiles, fileName) } return uploadedFiles, nil diff --git a/test/testenv/search_head_cluster_utils.go b/test/testenv/search_head_cluster_utils.go index 012064501..4dd326676 100644 --- a/test/testenv/search_head_cluster_utils.go +++ b/test/testenv/search_head_cluster_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,39 +18,11 @@ package testenv import ( "context" "fmt" - "os/exec" "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// DeleteSHC delete Search Head Cluster in given namespace -func DeleteSHC(ns string) { - output, err := exec.Command("kubectl", "delete", "shc", "-n", ns, "--all").Output() - if err != nil { - cmd := fmt.Sprintf("kubectl delete shc -n %s --all", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - } else { - logf.Log.Info("SHC deleted", "Namespace", ns, "stdout", output) - } -} - -// SHCInNamespace returns true if SHC is present in namespace -func SHCInNamespace(ns string) bool { - output, err := exec.Command("kubectl", "get", "searchheadcluster", "-n", ns).Output() - deleted := true - if err != nil { - cmd := fmt.Sprintf("kubectl get shc -n %s", ns) - logf.Log.Error(err, "Failed to execute command", "command", cmd) - return deleted - } - logf.Log.Info("Output of command", "Output", string(output)) - if strings.Contains(string(output), "No resources found in default namespace") { - deleted = false - } - return deleted -} - // DeployerAppChecksum Get the checksum for each app on the deployer func DeployerAppChecksum(ctx context.Context, deployment *Deployment) map[string]string { appChecksum := make(map[string]string) @@ -143,7 +115,7 @@ func DeployerBundlePushstatus(ctx context.Context, deployment *Deployment, ns st } for appName := range appChecksum { if _, present := appBundlePush[appName]; !present { - logf.Log.Info("Deployer app not found on any members", "Appname", appName) + logf.Log.Info("Deployer app not found on any members", "appName", appName) return make(map[string]int) } } diff --git a/test/testenv/search_utils.go b/test/testenv/search_utils.go index b0e85cc66..e6856d8dc 100644 --- a/test/testenv/search_utils.go +++ b/test/testenv/search_utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -77,7 +77,7 @@ func PerformSearchSync(ctx context.Context, podName string, search string, deplo return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", searchReqResp) + logf.Log.Info("Output of search Query", "search", search, "output", searchReqResp) // Since results can have multiple formats depending on the search SPL, leave this response as a string return searchReqResp, err @@ -102,7 +102,7 @@ func PerformSearchReq(ctx context.Context, podName string, search string, deploy return "", err } - logf.Log.Info("Output of search Query", "Search", search, "Output", stdout) + logf.Log.Info("Output of search Query", "search", search, "output", stdout) // Get SID var searchReqResult map[string]interface{} diff --git a/test/testenv/secretutil.go b/test/testenv/secretutil.go index 1b9ee9ff1..f30e72446 100644 --- a/test/testenv/secretutil.go +++ b/test/testenv/secretutil.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,14 +46,14 @@ func GetSecretStruct(ctx context.Context, deployment *Deployment, ns string, sec secretObject := &corev1.Secret{} err := deployment.GetInstance(ctx, secretName, secretObject) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get secret object", "Secret Name", secretName, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get secret object", "secretName", secretName, "namespace", ns) } return secretObject, err } // ModifySecretObject Modifies the secret object with given data func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string, data map[string][]byte) error { - logf.Log.Info("Modify secret object", "Secret Name", secretName, "Data", data) + logf.Log.Info("Modify secret object", "secretName", secretName, "data", data) secret := newSecretSpec(ns, secretName, data) err := deployment.UpdateCR(ctx, secret) @@ -66,7 +66,7 @@ func ModifySecretObject(ctx context.Context, deployment *Deployment, ns string, // DeleteSecretObject Deletes the entire secret object func DeleteSecretObject(ctx context.Context, deployment *Deployment, ns string, secretName string) error { - logf.Log.Info("Delete secret object", "Secret Name", secretName, "Namespace", ns) + logf.Log.Info("Delete secret object", "secretName", secretName, "namespace", ns) secret := newSecretSpec(ns, secretName, map[string][]byte{}) err := deployment.DeleteCR(ctx, secret) if err != nil { @@ -85,12 +85,12 @@ func GetMountedKey(ctx context.Context, deployment *Deployment, podName string, logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "" } - logf.Log.Info("Key found on pod", "Pod Name", podName, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Key found on pod", "podName", podName, "stdout", stdout, "stderr", stderr) return stdout } -// GetRandomeHECToken generates a random HEC token -func GetRandomeHECToken() string { +// GetRandomHECToken generates a random HEC token +func GetRandomHECToken() string { return fmt.Sprintf("%s-%s-%s-%s-%s", strings.ToUpper(RandomDNSName(8)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(4)), strings.ToUpper(RandomDNSName(12))) } @@ -99,7 +99,7 @@ func GetSecretFromServerConf(ctx context.Context, deployment *Deployment, podNam filePath := "/opt/splunk/etc/system/local/server.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } @@ -120,7 +120,7 @@ func DecryptSplunkEncodedSecret(ctx context.Context, deployment *Deployment, pod } logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) - logf.Log.Info("Decrypted Key Value", "Decrypted Key", stdout) + logf.Log.Info("Decrypted Key Value", "decryptedKey", stdout) return strings.TrimSuffix(stdout, "\n") } @@ -157,7 +157,7 @@ func GetVersionedSecretNames(ns string, version int) []string { } } } - logf.Log.Info("Versioned Secret Objects Found in Namespace", "NameSpace", ns, "Versioned Secrets", splunkSecrets) + logf.Log.Info("Versioned Secret Objects Found in Namespace", "namespace", ns, "versionedSecrets", splunkSecrets) return splunkSecrets } @@ -198,7 +198,7 @@ func GetSecretFromInputsConf(deployment *Deployment, podName string, ns string, filePath := "/opt/splunk/etc/apps/splunk_httpinput/local/inputs.conf" confline, err := GetConfLineFromPod(podName, filePath, ns, configName, stanza, true) if err != nil { - logf.Log.Error(err, "Failed to get secret from pod", "Pod Name", podName, "Secret Name", configName) + logf.Log.Error(err, "Failed to get secret from pod", "podName", podName, "secretName", configName) return "", "", err } secretList := strings.Split(confline, "=") diff --git a/test/testenv/test_setup_helpers.go b/test/testenv/test_setup_helpers.go new file mode 100644 index 000000000..326199d7d --- /dev/null +++ b/test/testenv/test_setup_helpers.go @@ -0,0 +1,245 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testenv + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/joho/godotenv" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" +) + +// SetupTestCaseEnv creates a new test case environment and deployment for use in BeforeEach blocks. +// It also validates test prerequisites immediately to fail fast before any long operations. +func SetupTestCaseEnv(testenvInstance *TestEnv, namePrefix string) (*TestCaseEnv, *Deployment) { + name := fmt.Sprintf("%s-%s", namePrefix+testenvInstance.GetName(), RandomDNSName(3)) + testcaseEnvInst, err := NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) + Expect(err).To(Succeed(), "Unable to create testcaseenv") + + deployment, err := testcaseEnvInst.NewDeployment(RandomDNSName(3)) + Expect(err).To(Succeed(), "Unable to create deployment") + + err = testcaseEnvInst.ValidateTestPrerequisites(context.TODO(), deployment) + Expect(err).To(Succeed(), "Test prerequisites validation failed") + + return testcaseEnvInst, deployment +} + +// TeardownTestCaseEnv handles the common teardown logic for test case environments. +func TeardownTestCaseEnv(testcaseEnvInst *TestCaseEnv, deployment *Deployment) { + if types.SpecState(ginkgo.CurrentSpecReport().State) == types.SpecStateFailed { + if testcaseEnvInst != nil { + testcaseEnvInst.SkipTeardown = true + } + } + + if deployment != nil { + deployment.Teardown() + } + + if testcaseEnvInst != nil { + Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) + } +} + +// CleanupOperatorFile deletes the test_file.img from the operator pod's app download directory +// if filePresentOnOperator is true. +func CleanupOperatorFile(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, filePresentOnOperator bool) { + if filePresentOnOperator { + opPod := GetOperatorPodName(testcaseEnvInst) + podDownloadPath := filepath.Join(AppDownloadVolume, "test_file.img") + DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) + } +} + +// TeardownAppFrameworkTestCaseEnv handles teardown for app framework tests with provider-specific +// cloud storage cleanup. cloudCleanup is called only if SkipTeardown is false. +func TeardownAppFrameworkTestCaseEnv(ctx context.Context, testcaseEnvInst *TestCaseEnv, deployment *Deployment, cloudCleanup func(), filePresentOnOperator bool) { + TeardownTestCaseEnv(testcaseEnvInst, deployment) + + if testcaseEnvInst != nil && !testcaseEnvInst.SkipTeardown && cloudCleanup != nil { + cloudCleanup() + } + + CleanupOperatorFile(ctx, deployment, testcaseEnvInst, filePresentOnOperator) +} + +// S3CloudCleanup returns a cleanup function that deletes the given files from an S3 bucket. +func S3CloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnS3(bucket, uploadedApps) + } +} + +// AzureCloudCleanup returns a cleanup function that deletes the given files from Azure Blob storage. +func AzureCloudCleanup(ctx context.Context, uploadedApps []string) func() { + return func() { + azureBlobClient := &AzureBlobClient{} + azureBlobClient.DeleteFilesOnAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, uploadedApps) + } +} + +// GCPCloudCleanup returns a cleanup function that deletes the given files from a GCP bucket. +func GCPCloudCleanup(bucket string, uploadedApps []string) func() { + return func() { + DeleteFilesOnGCP(bucket, uploadedApps) + } +} + +// LoadEnvFile traverses up the directory tree from the current working directory +// to find and load a .env file using godotenv. Returns nil if no .env file is found. +func LoadEnvFile() error { + dir, err := os.Getwd() + if err != nil { + return err + } + + for { + envFile := filepath.Join(dir, ".env") + if _, err := os.Stat(envFile); err == nil { + return godotenv.Load(envFile) + } + + parentDir := filepath.Dir(dir) + if parentDir == dir { + return nil + } + dir = parentDir + } +} + +// SetupS3AppsSuite initialises the test environment and, when running on EKS, +// downloads the V1 and V2 app sets from S3. +func SetupS3AppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "eks" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + err = DownloadFilesFromS3(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromS3(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// CleanupLocalAppDownloads tears down the test environment and removes locally +// downloaded app directories after a suite run. +func CleanupLocalAppDownloads(testenvInst *TestEnv, dirs ...string) { + if testenvInst != nil { + Expect(testenvInst.Teardown()).ToNot(HaveOccurred()) + } + for _, dir := range dirs { + Expect(os.RemoveAll(dir)).To(Succeed(), "Unable to delete locally downloaded app files from "+dir) + } +} + +// SetupAzureAppsSuite initialises the test environment and, when running on Azure, +// downloads the V1 and V2 app sets from Azure Blob. +func SetupAzureAppsSuite(suiteName, downloadDirV1, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "azure" { + ctx := context.TODO() + + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + containerName := "/test-data/appframework/v1apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV1, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + containerName = "/test-data/appframework/v2apps/" + err = DownloadFilesFromAzure(ctx, GetAzureEndpoint(ctx), StorageAccountKey, StorageAccount, downloadDirV2, containerName, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupGCPAppsSuite initialises the test environment and, when running on GCP, +// downloads the V1 and V2 app sets from GCS. +func SetupGCPAppsSuite(suiteName, testDataBucket, appDirV1, downloadDirV1, appDirV2, downloadDirV2 string) (*TestEnv, []string, []string) { + testenvInst, err := NewDefaultTestEnv(suiteName) + Expect(err).ToNot(HaveOccurred()) + + if ClusterProvider == "gcp" { + appListV1 := BasicApps + appFileList := GetAppFileList(appListV1) + + testenvInst.Log.Info("logging download details", "bucket", testDataBucket, "appDirV1", appDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) + err = DownloadFilesFromGCP(testDataBucket, appDirV1, downloadDirV1, appFileList) + Expect(err).To(Succeed(), "Unable to download V1 app files") + + appListV2 := append(appListV1, NewAppsAddedBetweenPolls...) + appFileList = GetAppFileList(appListV2) + + err = DownloadFilesFromGCP(testDataBucket, appDirV2, downloadDirV2, appFileList) + Expect(err).To(Succeed(), "Unable to download V2 app files") + + return testenvInst, appListV1, appListV2 + } + + testenvInst.Log.Info("Skipping Before Suite Setup", "provider", ClusterProvider) + return testenvInst, nil, nil +} + +// SetupLicenseConfigMap downloads the license file from the appropriate provider +// and creates a license config map. +func SetupLicenseConfigMap(ctx context.Context, testcaseEnvInst *TestCaseEnv) { + downloadDir := "licenseFolder" + var licenseFilePath string + var err error + + switch ClusterProvider { + case "eks": + licenseFilePath, err = DownloadLicenseFromS3Bucket() + Expect(err).To(Succeed(), "Unable to download license file from S3") + case "azure": + licenseFilePath, err = DownloadLicenseFromAzure(ctx, downloadDir) + Expect(err).To(Succeed(), "Unable to download license file from Azure") + case "gcp": + licenseFilePath, err = DownloadLicenseFromGCPBucket() + Expect(err).To(Succeed(), "Unable to download license file from GCP") + default: + testcaseEnvInst.Log.Info("Skipping license download", "provider", ClusterProvider) + return + } + + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) +} diff --git a/test/testenv/test_workflows.go b/test/testenv/test_workflows.go new file mode 100644 index 000000000..8c5d60245 --- /dev/null +++ b/test/testenv/test_workflows.go @@ -0,0 +1,131 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package testenv + +import ( + "context" + "fmt" + + . "github.com/onsi/gomega" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + corev1 "k8s.io/api/core/v1" +) + +// WorkflowResult contains the result of a workflow execution +type WorkflowResult struct { + Standalone *enterpriseApi.Standalone + ClusterManager *enterpriseApi.ClusterManager + IndexerCluster *enterpriseApi.IndexerCluster + SearchHeadCluster *enterpriseApi.SearchHeadCluster + MonitoringConsole *enterpriseApi.MonitoringConsole + LicenseManager *enterpriseApi.LicenseManager +} + +// RunStandaloneDeploymentWorkflow deploys a standalone instance and verifies it's ready +func RunStandaloneDeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) *WorkflowResult { + standalone := testcaseEnvInst.DeployAndVerifyStandalone(ctx, deployment, name, "", "") + + return &WorkflowResult{Standalone: standalone} +} + +// RunC3DeploymentWorkflow deploys a C3 cluster (CM + IDXC + SHC) and verifies all components are ready +func RunC3DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, mcRef string) *WorkflowResult { + err := deployment.DeploySingleSiteCluster(ctx, name, indexerReplicas, true, mcRef) + Expect(err).To(Succeed(), "Unable to deploy C3 cluster") + + testcaseEnvInst.VerifyClusterReadyAndRFSF(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM4DeploymentWorkflow deploys a M4 multisite cluster and verifies all components are ready +func RunM4DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int, mcRef string) *WorkflowResult { + err := deployment.DeployMultisiteClusterWithSearchHead(ctx, name, indexerReplicas, siteCount, mcRef) + Expect(err).To(Succeed(), "Unable to deploy M4 cluster") + + testcaseEnvInst.VerifyM4ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunM1DeploymentWorkflow deploys a M1 multisite indexer cluster (no SHC) and verifies components +func RunM1DeploymentWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int, siteCount int) *WorkflowResult { + err := deployment.DeployMultisiteCluster(ctx, name, indexerReplicas, siteCount, "") + Expect(err).To(Succeed(), "Unable to deploy M1 cluster") + + testcaseEnvInst.VerifyM1ClusterReady(ctx, deployment, siteCount, testcaseEnvInst.VerifyClusterManagerReady) + testcaseEnvInst.VerifyRFSFMet(ctx, deployment) + + return &WorkflowResult{} +} + +// RunStandaloneWithServiceAccountWorkflow deploys standalone with a service account +func RunStandaloneWithServiceAccountWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, serviceAccountName string) *WorkflowResult { + testcaseEnvInst.CreateServiceAccount(serviceAccountName) + + spec := enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + Image: testcaseEnvInst.GetSplunkImage(), + }, + Volumes: []corev1.Volume{}, + ServiceAccount: serviceAccountName, + }, + } + + standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, name, spec) + Expect(err).To(Succeed(), "Unable to deploy standalone with service account") + + testcaseEnvInst.VerifyStandaloneReady(ctx, deployment, name, standalone) + + standalonePodName := fmt.Sprintf(StandalonePod, name, 0) + testcaseEnvInst.VerifyServiceAccountConfiguredOnPod(deployment, testcaseEnvInst.GetName(), standalonePodName, serviceAccountName) + + return &WorkflowResult{Standalone: standalone} +} + +// RunDeleteStandaloneWorkflow deploys and deletes a standalone instance +func RunDeleteStandaloneWorkflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string) { + result := RunStandaloneDeploymentWorkflow(ctx, deployment, testcaseEnvInst, name) + + err := deployment.DeleteCR(ctx, result.Standalone) + Expect(err).To(Succeed(), "Unable to delete standalone instance") +} + +// RunDeleteC3Workflow deploys and deletes a C3 cluster +func RunDeleteC3Workflow(ctx context.Context, deployment *Deployment, testcaseEnvInst *TestCaseEnv, name string, indexerReplicas int) { + RunC3DeploymentWorkflow(ctx, deployment, testcaseEnvInst, name, indexerReplicas, "") + + idxc := &enterpriseApi.IndexerCluster{} + idxcName := name + "-idxc" + GetInstanceWithExpect(ctx, deployment, idxc, idxcName, "Unable to get Indexer Cluster instance") + + err := deployment.DeleteCR(ctx, idxc) + Expect(err).To(Succeed(), "Unable to delete Indexer Cluster") + + shc := &enterpriseApi.SearchHeadCluster{} + shcName := name + "-shc" + GetInstanceWithExpect(ctx, deployment, shc, shcName, "Unable to get Search Head Cluster instance") + + err = deployment.DeleteCR(ctx, shc) + Expect(err).To(Succeed(), "Unable to delete Search Head Cluster") + + cm := &enterpriseApi.ClusterManager{} + GetInstanceWithExpect(ctx, deployment, cm, name, "Unable to get Cluster Manager instance") + + err = deployment.DeleteCR(ctx, cm) + Expect(err).To(Succeed(), "Unable to delete Cluster Manager") +} diff --git a/test/testenv/testcaseenv.go b/test/testenv/testcaseenv.go index 4e0ccb3c4..baddffc17 100644 --- a/test/testenv/testcaseenv.go +++ b/test/testenv/testcaseenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -72,10 +72,8 @@ func NewDefaultTestCaseEnv(kubeClient client.Client, name string) (*TestCaseEnv, // NewTestCaseEnv creates a new test environment to run tests againsts func NewTestCaseEnv(kubeClient client.Client, name string, operatorImage string, splunkImage string, licenseFilePath string) (*TestCaseEnv, error) { - var envName string - // The name are used in various resource label and there is a 63 char limit. Do our part to make sure we do not exceed that limit - if len(envName) > 24 { + if len(name) > 24 { return nil, fmt.Errorf("name %s has exceeded 24 chars", name) } @@ -118,11 +116,6 @@ func (testenv *TestCaseEnv) GetSplunkImage() string { return testenv.splunkImage } -// IsOperatorInstalledClusterWide returns if operator is installed clusterwide -func (testenv *TestCaseEnv) IsOperatorInstalledClusterWide() string { - return testenv.clusterWideOperator -} - func (testenv *TestCaseEnv) setup() error { testenv.Log.Info("testenv initializing.\n") @@ -362,7 +355,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { operator := &appsv1.Deployment{} err = testenv.GetKubeClient().Get(ctx, namespacedName, operator) if err != nil { - testenv.Log.Error(err, "Unable to get operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to get operator", "operatorName", testenv.operatorName) return err } @@ -387,7 +380,7 @@ func (testenv *TestCaseEnv) attachPVCToOperator(name string) error { // update the operator deployment now err = testenv.GetKubeClient().Update(ctx, operator) if err != nil { - testenv.Log.Error(err, "Unable to update operator", "operator name", testenv.operatorName) + testenv.Log.Error(err, "Unable to update operator", "operatorName", testenv.operatorName) return err } diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index 83576beda..e718ca9e4 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,7 +70,7 @@ const ( // C3 deployments require bundle push across all indexers and SHC deployer which can exceed 5 minutes. AppInstallTimeout = 10 * time.Minute - // SearchHeadPod Template String for search head pod + // SearchHeadPod Template String for Search Head pod SearchHeadPod = "splunk-%s-shc-search-head-%d" // DeployerPod Template String for deployer pod @@ -79,10 +79,10 @@ const ( // StandalonePod Template String for standalone pod StandalonePod = "splunk-%s-standalone-%d" - // LicenseManagerPod Template String for standalone pod + // LicenseManagerPod Template String for License Manager pod LicenseManagerPod = "splunk-%s-license-manager-%d" - // LicenseMasterPod Template String for standalone pod + // LicenseMasterPod Template String for License Master pod LicenseMasterPod = "splunk-%s-" + splcommon.LicenseManager + "-%d" // IngestorPod Template String for ingestor pod @@ -132,11 +132,17 @@ const ( // ClusterMasterServiceName Cluster Master Service Template String ClusterMasterServiceName = "splunk-%s-cluster-master-service" - // DeployerServiceName Cluster Manager Service Template String + // DeployerServiceName Deployer Service Template String DeployerServiceName = "splunk-%s-shc-deployer-service" // CRUpdateRetryCount if CR Update fails retry these many time CRUpdateRetryCount = 10 + + // LogLineCount is the default number of log lines to ingest for test data + LogLineCount = 2000 + + // DefaultIngestIndex is the default index name used for test data ingestion + DefaultIngestIndex = "main" ) var ( diff --git a/test/testenv/util.go b/test/testenv/util.go index 90460849b..6d4032eb0 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,6 @@ import ( "os" "os/exec" "path" - "reflect" - "sort" "strings" "time" @@ -743,7 +741,7 @@ func newStandaloneWithSpec(name, ns string, spec enterpriseApi.StandaloneSpec) * return &new } -// newMonitoringConsoleSpec returns MC Spec with given name, namespace and license manager Ref +// newMonitoringConsoleSpec returns MC Spec with given name, namespace and License Manager Ref func newMonitoringConsoleSpec(name, ns, LicenseManagerRef, splunkImage string) *enterpriseApi.MonitoringConsole { licenseMasterRef, licenseManagerRef := swapLicenseManager(name, LicenseManagerRef) @@ -965,7 +963,7 @@ func ExecuteCommandOnPod(ctx context.Context, deployment *Deployment, podName st logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -977,7 +975,7 @@ func ExecuteCommandOnOperatorPod(ctx context.Context, deployment *Deployment, po logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "shell", command, "command", stdin, "error", err.Error()) return "", err } - logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } @@ -986,7 +984,7 @@ func GetConfigMap(ctx context.Context, deployment *Deployment, ns string, config configMap := &corev1.ConfigMap{} err := deployment.GetInstance(ctx, configMapName, configMap) if err != nil { - deployment.testenv.Log.Error(err, "Unable to get config map", "Config Map Name", configMap, "Namespace", ns) + deployment.testenv.Log.Error(err, "Unable to get config map", "configMapName", configMap, "namespace", ns) } return configMap, err } @@ -1057,7 +1055,7 @@ func newLicenseManagerWithGivenSpec(name, ns string, spec enterpriseApi.LicenseM return &new } -// newLicenseMasterWithGivenSpec create and initializes CR for License Manager Kind with Given Spec +// newLicenseMasterWithGivenSpec create and initializes CR for License Master Kind with Given Spec func newLicenseMasterWithGivenSpec(name, ns string, spec enterpriseApiV3.LicenseMasterSpec) *enterpriseApiV3.LicenseMaster { new := enterpriseApiV3.LicenseMaster{ TypeMeta: metav1.TypeMeta{ @@ -1115,19 +1113,9 @@ func GetDirsOrFilesInPath(ctx context.Context, deployment *Deployment, podName s return strings.Fields(stdout), err } -// CompareStringSlices checks if two string slices are matching -func CompareStringSlices(stringOne []string, stringTwo []string) bool { - if len(stringOne) != len(stringTwo) { - return false - } - sort.Strings(stringOne) - sort.Strings(stringTwo) - return reflect.DeepEqual(stringOne, stringTwo) -} - // CheckStringInSlice check if string is present in a slice func CheckStringInSlice(stringSlice []string, compString string) bool { - logf.Log.Info("Checking for string in slice", "String", compString, "String Slice", stringSlice) + logf.Log.Info("Checking for string in slice", "string", compString, "stringSlice", stringSlice) for _, item := range stringSlice { if strings.Contains(item, compString) { return true @@ -1170,16 +1158,6 @@ func GetPodUIDs(ns string) map[string]string { return splunkPodUIDs } -// DeletePod Delete pod in the namespace -func DeletePod(ns string, podName string) error { - _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() - if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) - return err - } - return nil -} - // DeleteOperatorPod Delete Operator Pod in the namespace func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { var podName string @@ -1193,7 +1171,7 @@ func DeleteOperatorPod(testcaseEnvInst *TestCaseEnv) error { _, err := exec.Command("kubectl", "delete", "pod", "-n", ns, podName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete operator pod ", "PodName", podName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete operator pod ", "podName", podName, "namespace", ns) return err } return nil @@ -1205,7 +1183,7 @@ func DeleteFilesOnOperatorPod(ctx context.Context, deployment *Deployment, podNa cmd := fmt.Sprintf("rm -f %s", filepath) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to delete file on pod ", "PodName", podName, "location", filepath, "command", cmd) + logf.Log.Error(err, "Failed to delete file on pod ", "podName", podName, "location", filepath, "command", cmd) return err } } @@ -1220,9 +1198,9 @@ func DumpGetSplunkVersion(ctx context.Context, ns string, deployment *Deployment if strings.Contains(podName, filterString) { stdout, err := ExecuteCommandOnPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to get splunkd version on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get splunkd version on the pod", "podName", podName) } - logf.Log.Info("Splunk Version Found", "Pod Name", podName, "Version", string(stdout)) + logf.Log.Info("Splunk Version Found", "podName", podName, "version", string(stdout)) } } } @@ -1232,7 +1210,7 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN cmd := fmt.Sprintf("cd %s && dd if=/dev/zero of=./%s bs=4k iflag=fullblock,count_bytes count=%s", filepath, filename, size) _, err := ExecuteCommandOnOperatorPod(ctx, deployment, podName, cmd) if err != nil { - logf.Log.Error(err, "Failed to create file on the pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to create file on the pod", "podName", podName) return err } return nil @@ -1240,10 +1218,10 @@ func CreateDummyFileOnOperator(ctx context.Context, deployment *Deployment, podN // DeleteConfigMap Delete configMap in the namespace func DeleteConfigMap(ns string, ConfigMapName string) error { - logf.Log.Info("Delete configMap", "configMap Name", ConfigMapName) + logf.Log.Info("Delete configMap", "configMapName", ConfigMapName) _, err := exec.Command("kubectl", "delete", "configmap", "-n", ns, ConfigMapName).Output() if err != nil { - logf.Log.Error(err, "Failed to delete config Map", "ConfigMap Name", ConfigMapName, "Namespace", ns) + logf.Log.Error(err, "Failed to delete config Map", "configMapName", ConfigMapName, "namespace", ns) return err } return nil diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 4b3bc1331..a6750ba9d 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,7 +83,7 @@ func (testenv *TestCaseEnv) VerifyMonitoringConsoleReady(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "phase", monitoringConsole.Status.Phase) DumpGetPods(testenv.GetName()) return monitoringConsole.Status.Phase @@ -106,7 +106,7 @@ func (testenv *TestCaseEnv) VerifyStandaloneReady(ctx context.Context, deploymen // Refresh the instance to get latest state err = deployment.GetInstance(ctx, standalone.Name, standalone) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + testenv.Log.Info("Standalone reached Ready phase", "instance", standalone.ObjectMeta.Name, "phase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around @@ -128,13 +128,13 @@ func (testenv *TestCaseEnv) VerifySearchHeadClusterReady(ctx context.Context, de shc := &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(ctx, instanceName, shc) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase, "DeployerPhase", shc.Status.DeployerPhase) + testenv.Log.Info("SearchHeadCluster reached Ready phase", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase, "deployerPhase", shc.Status.DeployerPhase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), shc) - testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenv.Log.Info("Check for Consistency Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "phase", shc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-shc-") return shc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -151,13 +151,13 @@ func (testenv *TestCaseEnv) VerifySingleSiteIndexersReady(ctx context.Context, d idc := &enterpriseApi.IndexerCluster{} err = deployment.GetInstance(ctx, instanceName, idc) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("IndexerCluster reached Ready phase", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer instance's phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") return idc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) @@ -198,15 +198,15 @@ func (testenv *TestCaseEnv) VerifyClusterManagerReady(ctx context.Context, deplo cm := &enterpriseApi.ClusterManager{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("ClusterManager reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-manager should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "cluster-manager") - testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency cluster-manager phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) return cm.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } @@ -221,13 +221,13 @@ func (testenv *TestCaseEnv) VerifyClusterMasterReady(ctx context.Context, deploy cm := &enterpriseApiV3.ClusterMaster{} err = deployment.GetInstance(ctx, deployment.GetName(), cm) gomega.Expect(err).To(gomega.Succeed()) - testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("ClusterMaster reached Ready phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) DumpGetPods(testenv.GetName()) // In a steady state, cluster-master should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, deployment.GetName(), cm) - testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenv.Log.Info("Check for Consistency cluster-master phase to be ready", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase) return cm.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } @@ -246,7 +246,7 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetPods(testenv.GetName()) return idc.Status.Phase @@ -255,14 +255,14 @@ func (testenv *TestCaseEnv) VerifyIndexersReady(ctx context.Context, deployment // In a steady state, we should stay in Ready and not flip-flop around gomega.Consistently(func() enterpriseApi.Phase { _ = deployment.GetInstance(ctx, instanceName, idc) - testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenv.Log.Info("Check for Consistency indexer site instance phase to be ready", "instance", instanceName, "phase", idc.Status.Phase) DumpGetSplunkVersion(ctx, testenv.GetName(), deployment, "-idxc-indexer-") return idc.Status.Phase }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(enterpriseApi.PhaseReady)) } } -// VerifyIndexerClusterMultisiteStatus verify indexer Cluster is configured as multisite +// VerifyIndexerClusterMultisiteStatus verify Indexer Cluster is configured as multisite func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Context, deployment *Deployment, siteCount int) { siteIndexerMap := map[string][]string{} for site := 1; site <= siteCount; site++ { @@ -281,10 +281,10 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(ctx, podName, command, stdin, false) if err != nil { - testenv.Log.Error(err, "Failed to execute command", "on pod", podName, "command", command) + testenv.Log.Error(err, "Failed to execute command", "onPod", podName, "command", command) return map[string][]string{} } - testenv.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + testenv.Log.Info("Command executed", "onPod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) siteIndexerResponse := ClusterManagerSitesResponse{} json.Unmarshal([]byte(stdout), &siteIndexerResponse) siteIndexerStatus := map[string][]string{} @@ -298,33 +298,24 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterMultisiteStatus(ctx context.Cont }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(siteIndexerMap)) } -// VerifyRFSFMet verify RF SF is met on cluster manager +// VerifyRFSFMet verify RF SF is met on Cluster Manager func (testenv *TestCaseEnv) VerifyRFSFMet(ctx context.Context, deployment *Deployment) { gomega.Eventually(func() bool { rfSfStatus := CheckRFSF(ctx, deployment) - testenv.Log.Info("Verifying RF SF is met", "Status", rfSfStatus) + testenv.Log.Info("Verifying RF SF is met", "status", rfSfStatus) return rfSfStatus }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) } -// VerifyNoDisconnectedSHPresentOnCM is present on cluster manager +// VerifyNoDisconnectedSHPresentOnCM verifies no disconnected SH is present on Cluster Manager func (testenv *TestCaseEnv) VerifyNoDisconnectedSHPresentOnCM(ctx context.Context, deployment *Deployment) { gomega.Consistently(func() bool { shStatus := CheckSearchHeadRemoved(ctx, deployment) - testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "Status", shStatus) + testenv.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "status", shStatus) return shStatus }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) } -// VerifyNoSHCInNamespace verify no SHC is present in namespace -func (testenv *TestCaseEnv) VerifyNoSHCInNamespace(deployment *Deployment) { - gomega.Eventually(func() bool { - shcStatus := SHCInNamespace(testenv.GetName()) - testenv.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) - return shcStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(false)) -} - // VerifyLicenseManagerReady verify LM is in ready status and does not flip flop func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deployment *Deployment) { LicenseManager := &enterpriseApi.LicenseManager{} @@ -336,7 +327,7 @@ func (testenv *TestCaseEnv) VerifyLicenseManagerReady(ctx context.Context, deplo return enterpriseApi.PhaseError } testenv.Log.Info("Waiting for License Manager instance status to be ready", - "instance", LicenseManager.ObjectMeta.Name, "Phase", LicenseManager.Status.Phase) + "instance", LicenseManager.ObjectMeta.Name, "phase", LicenseManager.Status.Phase) DumpGetPods(testenv.GetName()) return LicenseManager.Status.Phase @@ -360,7 +351,7 @@ func (testenv *TestCaseEnv) VerifyLicenseMasterReady(ctx context.Context, deploy return enterpriseApi.PhaseError } testenv.Log.Info("Waiting for License Master instance status to be ready", - "instance", LicenseMaster.ObjectMeta.Name, "Phase", LicenseMaster.Status.Phase) + "instance", LicenseMaster.ObjectMeta.Name, "phase", LicenseMaster.Status.Phase) DumpGetPods(testenv.GetName()) return LicenseMaster.Status.Phase @@ -393,7 +384,7 @@ func (testenv *TestCaseEnv) VerifyServiceAccountConfiguredOnPod(deployment *Depl restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { - testenv.Log.Error(err, "Failed to parse cluster Search heads") + testenv.Log.Error(err, "Failed to parse cluster Search Heads") return false } testenv.Log.Info("Service Account on Pod", "found", restResponse.Spec.ServiceAccount, "expected", serviceAccount) @@ -439,15 +430,6 @@ func (testenv *TestCaseEnv) VerifyIndexExistsOnS3(ctx context.Context, deploymen }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } -// VerifyRollingRestartFinished verify no rolling restart is active -func (testenv *TestCaseEnv) VerifyRollingRestartFinished(ctx context.Context, deployment *Deployment) { - gomega.Eventually(func() bool { - rollingRestartStatus := CheckRollingRestartStatus(ctx, deployment) - testenv.Log.Info("Rolling Restart Status", "active", rollingRestartStatus) - return rollingRestartStatus - }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(true)) -} - // VerifyConfOnPod Verify give conf and value on config file on pod func (testenv *TestCaseEnv) VerifyConfOnPod(deployment *Deployment, podName string, confFilePath string, config string, value string) { gomega.Consistently(func() bool { @@ -474,7 +456,7 @@ func (testenv *TestCaseEnv) VerifySearchHeadClusterPhase(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "Expected", phase, "Phase", shc.Status.Phase) + testenv.Log.Info("Waiting for Search Head Cluster Phase", "instance", shc.ObjectMeta.Name, "expected", phase, "phase", shc.Status.Phase) DumpGetPods(testenv.GetName()) return shc.Status.Phase @@ -489,7 +471,7 @@ func (testenv *TestCaseEnv) VerifyIndexerClusterPhase(ctx context.Context, deplo if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "Expected", phase, "Phase", idxc.Status.Phase) + testenv.Log.Info("Waiting for Indexer Cluster Phase", "instance", idxc.ObjectMeta.Name, "expected", phase, "phase", idxc.Status.Phase) DumpGetPods(testenv.GetName()) return idxc.Status.Phase @@ -504,7 +486,7 @@ func (testenv *TestCaseEnv) VerifyStandalonePhase(ctx context.Context, deploymen if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + testenv.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "expected", phase, "actualPhase", standalone.Status.Phase) DumpGetPods(testenv.GetName()) return standalone.Status.Phase @@ -519,7 +501,7 @@ func (testenv *TestCaseEnv) VerifyMonitoringConsolePhase(ctx context.Context, de if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) + testenv.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "expected", phase, "actualPhase", mc.Status.Phase) DumpGetPods(testenv.GetName()) return mc.Status.Phase @@ -614,7 +596,7 @@ func (testenv *TestCaseEnv) VerifyCustomResourceVersionChanged(ctx context.Conte if err != nil { return "-1" } - testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "Not Expected", resourceVersion, " Actual Resource Version", newResourceVersion) + testenv.Log.Info("Waiting for ", kind, " CR status", "instance", name, "notExpected", resourceVersion, "actualResourceVersion", newResourceVersion) DumpGetPods(testenv.GetName()) return newResourceVersion @@ -692,7 +674,7 @@ func (testenv *TestCaseEnv) VerifyResourceConstraints(deployment *Deployment, po }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } -// VerifyClusterManagerPhase verify phase of cluster manager +// VerifyClusterManagerPhase verify phase of Cluster Manager func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { cm := &enterpriseApi.ClusterManager{} gomega.Eventually(func() enterpriseApi.Phase { @@ -700,7 +682,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deplo if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Manager Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) // Test ClusterManager Phase to see if its ready @@ -708,7 +690,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerPhase(ctx context.Context, deplo }, deployment.GetTimeout(), ShortPollInterval).Should(gomega.Equal(phase)) } -// VerifyClusterMasterPhase verify phase of cluster manager +// VerifyClusterMasterPhase verify phase of Cluster Master func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deployment *Deployment, phase enterpriseApi.Phase) { cm := &enterpriseApiV3.ClusterMaster{} gomega.Eventually(func() enterpriseApi.Phase { @@ -716,7 +698,7 @@ func (testenv *TestCaseEnv) VerifyClusterMasterPhase(ctx context.Context, deploy if err != nil { return enterpriseApi.PhaseError } - testenv.Log.Info("Waiting for cluster-manager Phase", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase, "Expected", phase) + testenv.Log.Info("Waiting for Cluster Master Phase", "instance", cm.ObjectMeta.Name, "phase", cm.Status.Phase, "expected", phase) DumpGetPods(testenv.GetName()) // Test ClusterManager Phase to see if its ready @@ -734,9 +716,9 @@ func (testenv *TestCaseEnv) VerifySecretsOnPods(ctx context.Context, deployment comparsion := bytes.Compare([]byte(currentValue), secretValue) if comparsion == 0 { found = true - testenv.Log.Info("Secret Values on POD Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } else { - testenv.Log.Info("Secret Values on POD DONOT Match", "Match Expected", match, "Pod Name ", pod, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", currentValue) + testenv.Log.Info("Secret Values on POD DONOT Match", "matchExpected", match, "podName", pod, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", currentValue) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -754,10 +736,10 @@ func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, de secretValueOnSecretObject := currentSecretData.Data[secretKey] comparsion := bytes.Compare(secretValueOnSecretObject, secretValue) if comparsion == 0 { - testenv.Log.Info("Secret Values on Secret Object Match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object Match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) found = true } else { - testenv.Log.Info("Secret Values on Secret Object DONOT match", "Match Expected", match, "Secret Object Name", secretName, "Secret Key", secretKey, "Given Value of Key", string(secretValue), "Key Value found", string(secretValueOnSecretObject)) + testenv.Log.Info("Secret Values on Secret Object DONOT match", "matchExpected", match, "secretObjectName", secretName, "secretKey", secretKey, "givenValue", string(secretValue), "foundValue", string(secretValueOnSecretObject)) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -769,7 +751,7 @@ func (testenv *TestCaseEnv) VerifySecretsOnSecretObjects(ctx context.Context, de func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, deployment *Deployment, verificationPods []string, data map[string][]byte, match bool) { for _, podName := range verificationPods { keysToMatch := GetKeysToMatch(podName) - testenv.Log.Info("Verificaton Keys Set", "Pod Name", podName, "Keys To Compare", keysToMatch) + testenv.Log.Info("Verificaton Keys Set", "podName", podName, "keysToCompare", keysToMatch) for _, secretName := range keysToMatch { found := false stanza := SecretKeytoServerConfStanza[secretName] @@ -777,10 +759,10 @@ func (testenv *TestCaseEnv) VerifySplunkServerConfSecrets(ctx context.Context, d gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on server.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on server.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -794,16 +776,16 @@ func (testenv *TestCaseEnv) VerifySplunkInputConfSecrets(deployment *Deployment, for _, podName := range verificationPods { if strings.Contains(podName, "standalone") || strings.Contains(podName, "indexer") { found := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) stanza := SecretKeytoServerConfStanza[secretName] _, value, err := GetSecretFromInputsConf(deployment, podName, testenv.GetName(), "token", stanza) gomega.Expect(err).To(gomega.Succeed(), "Secret not found in conf file", "Secret Name", secretName) comparsion := strings.Compare(value, string(data[secretName])) if comparsion == 0 { - testenv.Log.Info("Secret Values on input.conf Match", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf Match", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) found = true } else { - testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "Match Expected", match, "Pod Name", podName, "Secret Key", secretName, "Given Value of Key", string(data[secretName]), "Key Value found", value) + testenv.Log.Info("Secret Values on input.conf DONOT MATCH", "matchExpected", match, "podName", podName, "secretKey", secretName, "givenValue", string(data[secretName]), "foundValue", value) } gomega.Expect(found).Should(gomega.Equal(match)) } @@ -821,7 +803,7 @@ func (testenv *TestCaseEnv) VerifySplunkSecretViaAPI(ctx context.Context, deploy } for _, secretName := range keysToMatch { validKey := false - testenv.Log.Info("Key Verificaton", "Pod Name", podName, "Key", secretName) + testenv.Log.Info("Key Verificaton", "podName", podName, "key", secretName) validKey = CheckSecretViaAPI(ctx, deployment, podName, secretName, string(data[secretName])) gomega.Expect(validKey).Should(gomega.Equal(match)) } @@ -840,7 +822,7 @@ func (testenv *TestCaseEnv) VerifyPVC(deployment *Deployment, ns string, pvcName break } } - testenv.Log.Info("PVC Status Verified", "PVC", pvcName, "STATUS", pvcExists, "EXPECTED", expectedToExist) + testenv.Log.Info("PVC Status Verified", "pvc", pvcName, "status", pvcExists, "expected", expectedToExist) return pvcExists }, verificationTimeout, PollInterval).Should(gomega.Equal(expectedToExist)) } @@ -938,7 +920,7 @@ func (testenv *TestCaseEnv) VerifyAppsInFolder(ctx context.Context, deployment * gomega.Eventually(func() bool { // Using checkAppDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkAppDirecotry is FALSE appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkAppDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "pod", podName) for _, app := range apps { folderName := app + "/" found := CheckStringInSlice(appList, folderName) @@ -956,10 +938,10 @@ func (testenv *TestCaseEnv) VerifyAppsDownloadedOnContainer(ctx context.Context, for _, podName := range pods { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get apps on pod", "pod", podName) for _, app := range apps { found := CheckStringInSlice(appList, app) - testenv.Log.Info("Check App files present on the pod", "Pod Name", podName, "App Name", app, "directory", path, "Status", found) + testenv.Log.Info("Check App files present on the pod", "podName", podName, "appName", app, "directory", path, "status", found) gomega.Expect(found).Should(gomega.Equal(true)) } } @@ -972,7 +954,7 @@ func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnOperatorContainer(ctx cont gomega.Eventually(func() bool { appList, err := GetOperatorDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on operator pod", "Pod", podName) + testenv.Log.Error(err, "Unable to get apps on operator pod", "pod", podName) return true } found := CheckStringInSlice(appList, app+"_") @@ -990,7 +972,7 @@ func (testenv *TestCaseEnv) VerifyAppsPackageDeletedOnContainer(ctx context.Cont gomega.Eventually(func() bool { appList, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, false) if err != nil { - testenv.Log.Error(err, "Unable to get apps on pod", "Pod", podName) + testenv.Log.Error(err, "Unable to get apps on pod", "pod", podName) return true } found := CheckStringInSlice(appList, app+"_") @@ -1016,7 +998,7 @@ func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment * testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) return phase // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase should not be %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appState", appDeploymentInfo) return appDeploymentInfo.PhaseInfo.Phase }, deployment.GetTimeout(), PollInterval).ShouldNot(gomega.Equal(phase)) } @@ -1033,9 +1015,9 @@ func (testenv *TestCaseEnv) VerifyAppListPhase(ctx context.Context, deployment * testenv.Log.Info(fmt.Sprintf("App deployment info not found yet for app %s (CR %s/%s, AppSource %s), continuing to poll", appName, crKind, name, appSourceName)) return enterpriseApi.PhaseDownload // Continue polling } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "Actual Phase", appDeploymentInfo.PhaseInfo.Phase, "App Phase Status", appDeploymentInfo.PhaseInfo.Status, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected Phase %s", crKind, name, appName, phase), "actualPhase", appDeploymentInfo.PhaseInfo.Phase, "appPhaseStatus", appDeploymentInfo.PhaseInfo.Status, "appState", appDeploymentInfo) if appDeploymentInfo.PhaseInfo.Status != enterpriseApi.AppPkgInstallComplete { - testenv.Log.Info("Phase Install Not Complete.", "Phase Found", appDeploymentInfo.PhaseInfo.Phase, "Phase Status Found", appDeploymentInfo.PhaseInfo.Status) + testenv.Log.Info("Phase Install Not Complete.", "phaseFound", appDeploymentInfo.PhaseInfo.Phase, "phaseStatusFound", appDeploymentInfo.PhaseInfo.Status) return enterpriseApi.PhaseDownload } return appDeploymentInfo.PhaseInfo.Phase @@ -1071,7 +1053,7 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deploym mcConfigMap, err := GetMCConfigMap(ctx, deployment, testenv.GetName(), mcName) gomega.Expect(err).To(gomega.Succeed(), "Unable to get MC config map") for _, podName := range pods { - testenv.Log.Info("Checking for POD on MC Config Map", "POD Name", podName, "DATA", mcConfigMap.Data) + testenv.Log.Info("Checking for POD on MC Config Map", "podName", podName, "data", mcConfigMap.Data) gomega.Expect(expected).To(gomega.Equal(CheckPodNameInString(podName, mcConfigMap.Data[key])), "Verify Pod in MC Config Map. Pod Name %s.", podName) } } @@ -1079,7 +1061,7 @@ func (testenv *TestCaseEnv) VerifyPodsInMCConfigMap(ctx context.Context, deploym // VerifyPodsInMCConfigString checks if given pod names are present in given KEY of given MC's Config Map func (testenv *TestCaseEnv) VerifyPodsInMCConfigString(ctx context.Context, deployment *Deployment, pods []string, mcName string, expected bool, checkPodIP bool) { for _, podName := range pods { - testenv.Log.Info("Checking pod configured in MC POD Peers String", "Pod Name", podName) + testenv.Log.Info("Checking pod configured in MC POD Peers String", "podName", podName) var found bool if checkPodIP { podIP := GetPodIP(testenv.GetName(), podName) @@ -1100,7 +1082,7 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, clusterManagerBundleStatus = CMBundlePushstatus(ctx, deployment, previousBundleHash, "cmaster") } if len(clusterManagerBundleStatus) < replicas { - testenv.Log.Info("Bundle push on Pod not complete on all pods", "Pod with bundle push", clusterManagerBundleStatus) + testenv.Log.Info("Bundle push on Pod not complete on all pods", "podWithBundlePush", clusterManagerBundleStatus) return false } clusterPodNames := DumpGetPods(testenv.GetName()) @@ -1109,11 +1091,11 @@ func (testenv *TestCaseEnv) VerifyClusterManagerBundlePush(ctx context.Context, if strings.Contains(podName, "-indexer-") { if _, present := clusterManagerBundleStatus[podName]; present { if clusterManagerBundleStatus[podName] != "Up" { - testenv.Log.Info("Bundle push on Pod not complete", "Pod Name", podName, "Status", clusterManagerBundleStatus[podName]) + testenv.Log.Info("Bundle push on Pod not complete", "podName", podName, "status", clusterManagerBundleStatus[podName]) return false } } else { - testenv.Log.Info("Bundle push not found on pod", "Podname", podName) + testenv.Log.Info("Bundle push not found on pod", "podName", podName) return false } } @@ -1134,7 +1116,7 @@ func (testenv *TestCaseEnv) VerifyDeployerBundlePush(ctx context.Context, deploy } for appName, val := range deployerAppPushStatus { if val < replicas { - testenv.Log.Info("Bundle push not complete on all pods for", "App Name", appName, "Replicas with bundle push", val, "Expected replicas", replicas) + testenv.Log.Info("Bundle push not complete on all pods for", "appName", appName, "replicasWithBundlePush", val, "expectedReplicas", replicas) DumpGetPods(testenv.GetName()) return false @@ -1164,7 +1146,7 @@ func (testenv *TestCaseEnv) VerifyNoPodResetByUID(ctx context.Context, deploymen } } -// WaitForSplunkPodCleanup Wait for cleanup to happend +// WaitForSplunkPodCleanup Wait for cleanup to happen func (testenv *TestCaseEnv) WaitForSplunkPodCleanup(ctx context.Context, deployment *Deployment) { gomega.Eventually(func() int { testenv.Log.Info("Waiting for Splunk Pods to be deleted before running test") @@ -1193,21 +1175,21 @@ func (testenv *TestCaseEnv) VerifyAppRepoState(ctx context.Context, deployment * testenv.Log.Error(err, "Failed to get app deployment info") return 0 } - testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "Actual Value", appDeploymentInfo.RepoState, "App State", appDeploymentInfo) + testenv.Log.Info(fmt.Sprintf("App State found for CR %s NAME %s APP NAME %s Expected repo value %d", crKind, name, appName, repoValue), "actualValue", appDeploymentInfo.RepoState, "appState", appDeploymentInfo) return int(appDeploymentInfo.RepoState) }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(repoValue)) } // VerifyIsDeploymentInProgressFlagIsSet verify IsDeploymentInProgress flag is set to true func (testenv *TestCaseEnv) VerifyIsDeploymentInProgressFlagIsSet(ctx context.Context, deployment *Deployment, name string, crKind string) { - testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "CR NAME", name, "CR Kind", crKind) + testenv.Log.Info("Check IsDeploymentInProgress Flag is set", "crName", name, "crKind", crKind) gomega.Eventually(func() bool { isDeploymentInProgress, err := GetIsDeploymentInProgressFlag(ctx, deployment, testenv, name, crKind) if err != nil { testenv.Log.Error(err, "Failed to get isDeploymentInProgress Flag") return false } - testenv.Log.Info("IsDeploymentInProgress Flag status found", "CR NAME", name, "CR Kind", crKind, "IsDeploymentInProgress", isDeploymentInProgress) + testenv.Log.Info("IsDeploymentInProgress Flag status found", "crName", name, "crKind", crKind, "isDeploymentInProgress", isDeploymentInProgress) return isDeploymentInProgress }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(true)) } @@ -1218,7 +1200,7 @@ func (testenv *TestCaseEnv) VerifyFilesInDirectoryOnPod(ctx context.Context, dep gomega.Eventually(func() bool { // Using checkDirectory here to get all files in case of negative check. GetDirsOrFilesInPath will return files/directory when checkDirecotry is FALSE filelist, err := GetDirsOrFilesInPath(ctx, deployment, podName, path, checkDirectory) - gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "Pod", podName) + gomega.Expect(err).To(gomega.Succeed(), "Unable to get files on pod", "pod", podName) for _, file := range files { found := CheckStringInSlice(filelist, file) testenv.Log.Info("File check", "pod", podName, "filename", file, "path", path, "status", found)