diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 9d2aa1e..59fc8c1 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1016,7 +1016,37 @@ export async function abortRollout(serviceId: string) { .delete(deployments) .where(eq(deployments.rolloutId, inProgressRollout.id)); - await db.delete(workQueue).where(eq(workQueue.status, "pending")); + const pendingWork = await db + .select({ id: workQueue.id, payload: workQueue.payload }) + .from(workQueue) + .where( + and( + eq(workQueue.status, "pending"), + eq(workQueue.type, "deploy"), + inArray(workQueue.serverId, [...serverContainers.keys()]), + ), + ); + + const rolloutDeploymentIds = new Set(rolloutDeployments.map((d) => d.id)); + const workToDelete = pendingWork.filter((w) => { + try { + const parsed = JSON.parse(w.payload); + return rolloutDeploymentIds.has(parsed.deploymentId); + } catch { + return false; + } + }); + + if (workToDelete.length > 0) { + await db + .delete(workQueue) + .where( + inArray( + workQueue.id, + workToDelete.map((w) => w.id), + ), + ); + } return { success: true }; } diff --git a/web/components/service/details/deployment-canvas.tsx b/web/components/service/details/deployment-canvas.tsx index 39b79d0..600ef6f 100644 --- a/web/components/service/details/deployment-canvas.tsx +++ b/web/components/service/details/deployment-canvas.tsx @@ -305,27 +305,19 @@ export function DeploymentCanvas({ service }: DeploymentCanvasProps) { if (service.deployments.length === 0) { return ( - <> -
-
- -
-

No deployments yet.

-
- -
- -
-

No deployments yet.

+ +
+
- } - /> - +

No deployments yet.

+ + } + /> ); } @@ -335,26 +327,28 @@ export function DeploymentCanvas({ service }: DeploymentCanvasProps) { return ( <> -
- {hasEndpoints && ( - - )} - {serverGroups.map((group) => ( -
- +
+ {hasEndpoints && ( + - {hasVolumes && } -
- ))} -
+ )} + {serverGroups.map((group) => ( +
+ + {hasVolumes && } +
+ ))} +
+
-
{content}
+
+ {content &&
{content}
} +
); }); diff --git a/web/components/service/details/pending-changes-banner.tsx b/web/components/service/details/pending-changes-banner.tsx index 6d2e09e..f584ba3 100644 --- a/web/components/service/details/pending-changes-banner.tsx +++ b/web/components/service/details/pending-changes-banner.tsx @@ -67,7 +67,8 @@ export const PendingChangesBanner = memo(function PendingChangesBanner({ opacity: showBanner ? 1 : 0, }} > -
+
+
@@ -122,6 +123,7 @@ export const PendingChangesBanner = memo(function PendingChangesBanner({
+
); }); diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index f8e0f10..9cc72be 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -230,6 +230,10 @@ export async function applyStatusReport( } if (deployment.status === "pending" || deployment.status === "pulling") { + if (container.status !== "running") { + continue; + } + const service = await db .select() .from(services) @@ -306,6 +310,7 @@ export async function applyStatusReport( if ( deployment.status === "starting" && + container.status === "running" && (healthStatus === "healthy" || healthStatus === "none") ) { console.log( diff --git a/web/lib/email/index.ts b/web/lib/email/index.ts index b629199..a04cafa 100644 --- a/web/lib/email/index.ts +++ b/web/lib/email/index.ts @@ -259,7 +259,7 @@ export async function sendBuildFailureAlert( type DeploymentFailureAlertOptions = { serviceId: string; - serverId: string; + serverId: string | null; failedStage?: string; }; @@ -272,44 +272,79 @@ export async function sendDeploymentFailureAlert( return; } - const [result] = await db - .select({ - serviceName: services.name, - projectName: projects.name, - projectSlug: projects.slug, - envName: environments.name, - serverName: servers.name, - }) - .from(services) - .innerJoin(projects, eq(projects.id, services.projectId)) - .innerJoin(environments, eq(environments.id, services.environmentId)) - .innerJoin(servers, eq(servers.id, options.serverId)) - .where(eq(services.id, options.serviceId)); - - if (!result) { - return; + let serviceName: string; + let projectName: string; + let projectSlug: string; + let envName: string; + let serverName: string; + + if (options.serverId) { + const [result] = await db + .select({ + serviceName: services.name, + projectName: projects.name, + projectSlug: projects.slug, + envName: environments.name, + serverName: servers.name, + }) + .from(services) + .innerJoin(projects, eq(projects.id, services.projectId)) + .innerJoin(environments, eq(environments.id, services.environmentId)) + .innerJoin(servers, eq(servers.id, options.serverId)) + .where(eq(services.id, options.serviceId)); + + if (!result) { + return; + } + + serviceName = result.serviceName; + projectName = result.projectName; + projectSlug = result.projectSlug; + envName = result.envName; + serverName = result.serverName; + } else { + const [result] = await db + .select({ + serviceName: services.name, + projectName: projects.name, + projectSlug: projects.slug, + envName: environments.name, + }) + .from(services) + .innerJoin(projects, eq(projects.id, services.projectId)) + .innerJoin(environments, eq(environments.id, services.environmentId)) + .where(eq(services.id, options.serviceId)); + + if (!result) { + return; + } + + serviceName = result.serviceName; + projectName = result.projectName; + projectSlug = result.projectSlug; + envName = result.envName; + serverName = "Unknown"; } - const baseUrl = getAppBaseUrl(); const serviceUrl = baseUrl - ? `${baseUrl}/dashboard/projects/${result.projectSlug}/${result.envName}/services/${options.serviceId}` + ? `${baseUrl}/dashboard/projects/${projectSlug}/${envName}/services/${options.serviceId}` : undefined; const details = [ - { label: "Service", value: result.serviceName }, - { label: "Project", value: result.projectName }, - { label: "Server", value: result.serverName }, + { label: "Service", value: serviceName }, + { label: "Project", value: projectName }, + { label: "Server", value: serverName }, ...(options.failedStage ? [{ label: "Failed Stage", value: options.failedStage }] : []), ]; await sendAlert({ - subject: `Deployment Failed: ${result.serviceName}`, + subject: `Deployment Failed: ${serviceName}`, template: Alert({ bannerText: "DEPLOYMENT FAILED", heading: "Deployment Failure Alert", - description: `The deployment for service "${result.serviceName}" in project "${result.projectName}" has failed on server "${result.serverName}".`, + description: `The deployment for service "${serviceName}" in project "${projectName}" has failed on server "${serverName}".`, details, buttonText: serviceUrl ? "View Service" : undefined, buttonUrl: serviceUrl, diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index cfbd4d7..e67b32e 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -211,13 +211,6 @@ export async function prepareRollingUpdate( (d) => d.status === "running" || d.status === "healthy", ); - for (const dep of runningDeployments) { - await db - .update(deployments) - .set({ status: "draining" }) - .where(eq(deployments.id, dep.id)); - } - return { deploymentIds: runningDeployments.map((d) => d.id) }; } diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 6e915fa..186c35f 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -14,7 +14,24 @@ export async function handleRolloutFailure( .from(deployments) .where(eq(deployments.rolloutId, rolloutId)); - if (rolloutDeployments.length === 0) return; + await db + .update(rollouts) + .set({ status: "rolled_back", completedAt: new Date() }) + .where(eq(rollouts.id, rolloutId)); + + if (rolloutDeployments.length === 0) { + sendDeploymentFailureAlert({ + serviceId, + serverId: null, + failedStage: reason, + }).catch((error) => { + console.error( + "[rollout:failure] failed to send deployment failure alert:", + error, + ); + }); + return; + } const serverId = rolloutDeployments[0].serverId; @@ -41,16 +58,12 @@ export async function handleRolloutFailure( "pulling", "starting", "healthy", + "running", "failed", ]), ), ); - await db - .update(rollouts) - .set({ status: "rolled_back", completedAt: new Date() }) - .where(eq(rollouts.id, rolloutId)); - sendDeploymentFailureAlert({ serviceId, serverId, diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 5cc51ee..ae17eb2 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray, isNull, ne, or } from "drizzle-orm"; import { db } from "@/db"; import { deployments, rollouts } from "@/db/schema"; import { getService } from "@/db/queries"; @@ -246,21 +246,25 @@ export const rolloutWorkflow = inngest.createFunction( await db .update(deployments) - .set({ status: "stopping" }) + .set({ status: "running" }) .where( and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), + eq(deployments.rolloutId, rolloutId), + eq(deployments.status, "healthy"), ), ); await db .update(deployments) - .set({ status: "running" }) + .set({ status: "draining" }) .where( and( - eq(deployments.rolloutId, rolloutId), - eq(deployments.status, "healthy"), + eq(deployments.serviceId, serviceId), + inArray(deployments.status, ["running", "healthy"]), + or( + ne(deployments.rolloutId, rolloutId), + isNull(deployments.rolloutId), + ), ), ); @@ -282,6 +286,8 @@ export const rolloutWorkflow = inngest.createFunction( ), ); + const dnsTimedOut = dnsResults.some((r) => r === null); + for (let i = 0; i < dnsResults.length; i++) { if (dnsResults[i] === null) { console.warn( @@ -298,6 +304,38 @@ export const rolloutWorkflow = inngest.createFunction( } } + if (dnsTimedOut) { + await step.run("rollback-dns-timeout", async () => { + await handleRolloutFailure( + rolloutId, + serviceId, + "dns_sync_timeout", + isRollingUpdate, + ); + }); + return { status: "rolled_back", rolloutId, reason: "dns_sync_timeout" }; + } + + if (isRollingUpdate) { + await step.run("stop-old-deployments", async () => { + await db + .update(deployments) + .set({ status: "stopping" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "draining"), + ), + ); + await ingestRolloutLog( + rolloutId, + serviceId, + "dns_sync", + "Stopping old deployments after DNS sync", + ); + }); + } + await step.run("complete-rollout", async () => { await db .update(rollouts) diff --git a/web/package.json b/web/package.json index 591853a..5e451b2 100644 --- a/web/package.json +++ b/web/package.json @@ -3,7 +3,7 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev", + "dev": "concurrently -n next,inngest -c blue,magenta \"next dev\" \"npx inngest-cli@latest dev\"", "build": "next build", "start": "next start", "lint": "next lint", @@ -56,6 +56,7 @@ "@types/react": "19.2.7", "@types/react-dom": "19.2.3", "@types/validator": "^13.15.10", + "concurrently": "^9.2.1", "drizzle-kit": "^0.31.8", "eslint": "^9", "eslint-config-next": "16.1.1", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index c61bc23..6a902ca 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -132,6 +132,9 @@ importers: '@types/validator': specifier: ^13.15.10 version: 13.15.10 + concurrently: + specifier: ^9.2.1 + version: 9.2.1 drizzle-kit: specifier: ^0.31.8 version: 0.31.8 @@ -3242,6 +3245,11 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + concurrently@9.2.1: + resolution: {integrity: sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==} + engines: {node: '>=18'} + hasBin: true + content-disposition@1.0.1: resolution: {integrity: sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==} engines: {node: '>=18'} @@ -5102,6 +5110,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-array-concat@1.1.3: resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==} engines: {node: '>=0.4'} @@ -5178,6 +5189,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} + side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} @@ -5326,6 +5341,10 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} @@ -5391,6 +5410,10 @@ packages: tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + ts-api-utils@2.4.0: resolution: {integrity: sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==} engines: {node: '>=18.12'} @@ -9159,6 +9182,15 @@ snapshots: concat-map@0.0.1: {} + concurrently@9.2.1: + dependencies: + chalk: 4.1.2 + rxjs: 7.8.2 + shell-quote: 1.8.3 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + content-disposition@1.0.1: {} content-type@1.0.5: {} @@ -11107,6 +11139,10 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-array-concat@1.1.3: dependencies: call-bind: 1.0.8 @@ -11273,6 +11309,8 @@ snapshots: shebang-regex@3.0.0: {} + shell-quote@1.8.3: {} + side-channel-list@1.0.0: dependencies: es-errors: 1.3.0 @@ -11437,6 +11475,10 @@ snapshots: dependencies: has-flag: 4.0.0 + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + supports-preserve-symlinks-flag@1.0.0: {} swr@2.4.0(react@19.2.3): @@ -11488,6 +11530,8 @@ snapshots: tr46@0.0.3: {} + tree-kill@1.2.2: {} + ts-api-utils@2.4.0(typescript@5.9.3): dependencies: typescript: 5.9.3