test: grafana SLO

tests/grafana/amp-grafana.test.ts

@@ -0,0 +1,187 @@
+import { it } from 'node:test';
+import * as assert from 'node:assert';
+import * as studion from '@studion/infra-code-blocks';
+import {
+  GetRoleCommand,
+  GetRolePolicyCommand,
+  ListRolePoliciesCommand,
+} from '@aws-sdk/client-iam';
+import { Unwrap } from '@pulumi/pulumi';
+import { backOff } from '../util';
+import { GrafanaTestContext } from './test-context';
+import { grafanaRequest, requestEndpointWithExpectedStatus } from './util';
+
+const backOffConfig = { numOfAttempts: 15 };
+
+export function testAmpGrafana(ctx: GrafanaTestContext) {
+  it('should have created the AMP data source', async () => {
+    const grafana = ctx.outputs!.ampGrafana;
+    const ampDataSource = (
+      grafana.connections[0] as studion.grafana.AMPConnection
+    ).dataSource;
+    const ampDataSourceName = ampDataSource.name as unknown as Unwrap<
+      typeof ampDataSource.name
+    >;
+
+    await backOff(async () => {
+      const { body, statusCode } = await grafanaRequest(
+        ctx,
+        'GET',
+        `/api/datasources/name/${encodeURIComponent(ampDataSourceName)}`,
+      );
+      assert.strictEqual(statusCode, 200, 'Expected data source to exist');
+
+      const data = (await body.json()) as Record<string, unknown>;
+      assert.strictEqual(
+        data.type,
+        'grafana-amazonprometheus-datasource',
+        'Expected Amazon Prometheus data source type',
+      );
+
+      const workspace = ctx.outputs!.ampWorkspace;
+      const ampEndpoint = workspace.prometheusEndpoint as unknown as Unwrap<
+        typeof workspace.prometheusEndpoint
+      >;
+      assert.ok(
+        (data.url as string).includes(ampEndpoint.replace(/\/$/, '')),
+        'Expected data source URL to contain the AMP workspace endpoint',
+      );
+    }, backOffConfig);
+  });
+
+  it('should have created the dashboard with expected panels', async () => {
+    const dashboard = ctx.outputs!.ampGrafana.dashboards[0];
+    const dashboardUid = dashboard.uid as unknown as Unwrap<
+      typeof dashboard.uid
+    >;
+
+    await backOff(async () => {
+      const { body, statusCode } = await grafanaRequest(
+        ctx,
+        'GET',
+        `/api/dashboards/uid/${dashboardUid}`,
+      );
+      assert.strictEqual(statusCode, 200, 'Expected dashboard to exist');
+
+      const data = (await body.json()) as {
+        dashboard: { title: string; panels: Array<{ title: string }> };
+      };
+      assert.strictEqual(
+        data.dashboard.title,
+        'ICB Grafana Test SLO',
+        'Expected dashboard title to match',
+      );
+
+      const panelTitles = data.dashboard.panels.map(p => p.title).sort();
+      const expectedPanels = [
+        'Availability',
+        'Availability Burn Rate',
+        'Success Rate',
+        'Success Rate Burn Rate',
+        'HTTP Request Success Rate',
+        'Request % below 250ms',
+        'Latency Burn Rate',
+        '99th Percentile Latency',
+        'Request percentage below 250ms',
+      ];
+      assert.deepStrictEqual(
+        panelTitles,
+        expectedPanels.sort(),
+        'Dashboard panels do not match expected panels',
+      );
+    }, backOffConfig);
+  });
+
+  it('should display metrics data in the dashboard', async () => {
+    await requestEndpointWithExpectedStatus(ctx, ctx.config.usersPath, 200);
+
+    const ampDataSource = (
+      ctx.outputs!.ampGrafana.connections[0] as studion.grafana.AMPConnection
+    ).dataSource;
+    const ampDataSourceName = ampDataSource.name as unknown as Unwrap<
+      typeof ampDataSource.name
+    >;
+    const { body: dsBody } = await grafanaRequest(
+      ctx,
+      'GET',
+      `/api/datasources/name/${encodeURIComponent(ampDataSourceName)}`,
+    );
+    const dsData = (await dsBody.json()) as Record<string, unknown>;
+    const dataSourceUid = dsData.uid as string;
+
+    await backOff(async () => {
+      const { body, statusCode } = await grafanaRequest(
+        ctx,
+        'POST',
+        '/api/ds/query',
+        {
+          queries: [
+            {
+              datasource: {
+                type: 'grafana-amazonprometheus-datasource',
+                uid: dataSourceUid,
+              },
+              expr: `{__name__=~"${ctx.config.ampNamespace}_.*"}`,
+              instant: true,
+              refId: 'A',
+            },
+          ],
+          from: 'now-5m',
+          to: 'now',
+        },
+      );
+      assert.strictEqual(statusCode, 200, 'Expected query to succeed');
+
+      const data = (await body.json()) as {
+        results: Record<string, { frames: Array<unknown> }>;
+      };
+      const frames = data.results?.A?.frames ?? [];
+      assert.ok(
+        frames.length > 0,
+        `Expected Grafana to return metric frames for namespace '${ctx.config.ampNamespace}'`,
+      );
+    }, backOffConfig);
+  });
+
+  it('should have created the IAM role with AMP inline policy', async () => {
+    const iamRole = ctx.outputs!.ampGrafana.connections[0].role;
+    const grafanaAmpRoleArn = iamRole.arn as unknown as Unwrap<
+      typeof iamRole.arn
+    >;
+    const roleName = grafanaAmpRoleArn.split('/').pop()!;
+    const { Role } = await ctx.clients.iam.send(
+      new GetRoleCommand({ RoleName: roleName }),
+    );
+    assert.ok(Role, 'Grafana IAM role should exist');
+
+    const { PolicyNames } = await ctx.clients.iam.send(
+      new ListRolePoliciesCommand({ RoleName: roleName }),
+    );
+    assert.ok(
+      PolicyNames && PolicyNames.length > 0,
+      'IAM role should have at least one inline policy',
+    );
+
+    const { PolicyDocument } = await ctx.clients.iam.send(
+      new GetRolePolicyCommand({
+        RoleName: roleName,
+        PolicyName: PolicyNames[0],
+      }),
+    );
+    const policy = JSON.parse(decodeURIComponent(PolicyDocument!)) as {
+      Statement: Array<{ Action: string[] }>;
+    };
+    const actions = policy.Statement.flatMap(s => s.Action).sort();
+    const expectedActions = [
+      'aps:GetSeries',
+      'aps:GetLabels',
+      'aps:GetMetricMetadata',
+      'aps:QueryMetrics',
+    ].sort();
+    assert.deepStrictEqual(
+      actions,
+      expectedActions,
+      'AMP policy actions do not match expected actions',
+    );
+  });
+}

tests/grafana/configurable-grafana.test.ts

@@ -0,0 +1,66 @@
+import { it } from 'node:test';
+import * as assert from 'node:assert';
+import * as studion from '@studion/infra-code-blocks';
+import {
+  GetRoleCommand,
+  GetRolePolicyCommand,
+  ListRolePoliciesCommand,
+} from '@aws-sdk/client-iam';
+import { Unwrap } from '@pulumi/pulumi';
+import { backOff } from '../util';
+import { GrafanaTestContext } from './test-context';
+import { grafanaRequest } from './util';
+
+const backOffConfig = { numOfAttempts: 15 };
+
+export function testConfigurableGrafana(ctx: GrafanaTestContext) {
+  it('should have created the folder with the configured name', async () => {
+    const folder = ctx.outputs!.configurableGrafanaComponent.folder;
+    const folderUid = folder.uid as unknown as Unwrap<typeof folder.uid>;
+
+    await backOff(async () => {
+      const { body, statusCode } = await grafanaRequest(
+        ctx,
+        'GET',
+        `/api/folders/${folderUid}`,
+      );
+      assert.strictEqual(statusCode, 200, 'Expected folder to exist');
+
+      const data = (await body.json()) as { title: string };
+      assert.strictEqual(
+        data.title,
+        'ICB Configurable Test Folder',
+        'Expected folder title to match withFolderName() value',
+      );
+    }, backOffConfig);
+  });
+
+  it('should have created the custom dashboard', async () => {
+    const dashboard = ctx.outputs!.configurableGrafanaComponent.dashboards[0];
+    const dashboardUid = dashboard.uid as unknown as Unwrap<
+      typeof dashboard.uid
+    >;
+
+    await backOff(async () => {
+      const { body, statusCode } = await grafanaRequest(
+        ctx,
+        'GET',
+        `/api/dashboards/uid/${dashboardUid}`,
+      );
+      assert.strictEqual(statusCode, 200, 'Expected custom dashboard to exist');
+
+      const data = (await body.json()) as {
+        dashboard: { title: string; panels: Array<{ title: string }> };
+      };
+      assert.strictEqual(
+        data.dashboard.title,
+        'ICB Grafana Configurable Dashboard',
+        'Expected custom dashboard title',
+      );
+      assert.ok(
+        data.dashboard.panels.length > 0,
+        'Expected at least one panel',
+      );
+    }, backOffConfig);
+  });
+}

tests/grafana/index.test.ts

@@ -0,0 +1,42 @@
+import { before, describe, after } from 'node:test';
+import { InlineProgramArgs, OutputMap } from '@pulumi/pulumi/automation';
+import { IAMClient } from '@aws-sdk/client-iam';
+import * as automation from '../automation';
+import { requireEnv, unwrapOutputs } from '../util';
+import { testAmpGrafana } from './amp-grafana.test';
+import { testConfigurableGrafana } from './configurable-grafana.test';
+import * as infraConfig from './infrastructure/config';
+import { GrafanaTestContext, ProgramOutput } from './test-context';
+
+const programArgs: InlineProgramArgs = {
+  stackName: 'dev',
+  projectName: 'icb-test-grafana',
+  program: () => import('./infrastructure'),
+};
+
+const region = requireEnv('AWS_REGION');
+const ctx: GrafanaTestContext = {
+  config: {
+    region,
+    usersPath: infraConfig.usersPath,
+    appName: infraConfig.appName,
+    ampNamespace: infraConfig.ampNamespace,
+    grafanaUrl: requireEnv('GRAFANA_URL'),
+    grafanaAuth: requireEnv('GRAFANA_AUTH'),
+  },
+  clients: {
+    iam: new IAMClient({ region }),
+  },
+};
+
+describe('Grafana component deployment', () => {
+  before(async () => {
+    const outputs: OutputMap = await automation.deploy(programArgs);
+    ctx.outputs = unwrapOutputs<ProgramOutput>(outputs);
+  });
+
+  after(() => automation.destroy(programArgs));
+
+  describe('AMP Grafana', () => testAmpGrafana(ctx));
+  describe('Configurable Grafana', () => testConfigurableGrafana(ctx));
+});

tests/grafana/infrastructure/config.ts

@@ -0,0 +1,11 @@
+export const appName = 'grafana-test';
+
+export const appImage = 'studiondev/observability-sample-app';
+
+export const appPort = 3000;
+
+export const usersPath = '/users';
+
+export const ampNamespace = 'icb_grafana_integration';
+
+export const apiFilter = 'http_route=~"/.*"';