diff --git a/eng/pipelines/templates/common.yml b/eng/pipelines/templates/common.yml index 1f6abe2d93..41b45e7aad 100644 --- a/eng/pipelines/templates/common.yml +++ b/eng/pipelines/templates/common.yml @@ -132,7 +132,8 @@ extends: - template: /eng/pipelines/templates/variables/image.yml - template: /eng/pipelines/templates/variables/globals.yml jobs: - - template: /eng/pipelines/templates/jobs/release.yml + # TODO: Revert to release.yml after Docker ARM64 testing + - template: /eng/pipelines/templates/jobs/release-anu.yml parameters: ServerName: ${{ parameters.ServerName }} IncludeNative: false diff --git a/eng/pipelines/templates/jobs/docker/build-docker.yml b/eng/pipelines/templates/jobs/docker/build-docker.yml index 335c427ff8..c2eca3bf59 100644 --- a/eng/pipelines/templates/jobs/docker/build-docker.yml +++ b/eng/pipelines/templates/jobs/docker/build-docker.yml @@ -12,8 +12,8 @@ jobs: strategy: matrix: $[ ${{ parameters.ServerMatrix }}] pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) + name: $(Pool) + image: $(VMImage) os: linux steps: @@ -30,7 +30,7 @@ jobs: echo "Listing $(ArtifactPath) contents:" ls -la "$(Pipeline.Workspace)/docker_staging/$(ArtifactPath)/" echo "" - echo "Listing linux-x64 contents:" + echo "Listing $(Platform) contents:" ls -la "$(Pipeline.Workspace)/docker_staging/$(ArtifactPath)/$(Platform)" echo "" echo "Listing dist contents:" @@ -67,8 +67,8 @@ jobs: # Save image as .tar echo "Saving image" - docker save "$(DockerLocalTag)" -o "$(Build.ArtifactStagingDirectory)/$(CliName)-image.tar" - ls -lh "$(Build.ArtifactStagingDirectory)/$(CliName)-image.tar" + docker save "$(DockerLocalTag)" -o "$(Build.ArtifactStagingDirectory)/$(CliName)-$(Architecture)-image.tar" + ls -lh "$(Build.ArtifactStagingDirectory)/$(CliName)-$(Architecture)-image.tar" displayName: Capture image digest and save as .tar - task: 1ES.PublishPipelineArtifact@1 diff --git a/eng/pipelines/templates/jobs/docker/release-docker.yml b/eng/pipelines/templates/jobs/docker/release-docker.yml index a9ca474a1d..86265a3c24 100644 --- a/eng/pipelines/templates/jobs/docker/release-docker.yml +++ b/eng/pipelines/templates/jobs/docker/release-docker.yml @@ -63,45 +63,13 @@ jobs: Write-Host "##vso[task.setvariable variable=DockerImageName;isOutput=true]$dockerImageName" name: SetReleaseVariables - - script: | - set -euo pipefail - - # Load image from tar - echo "Loading image tar for ${{ parameters.ServerName }}" - LOADED_IMAGE=$(docker load -i $(Pipeline.Workspace)/docker_output/$(SetReleaseVariables.CliName)-image.tar | grep -oP 'Loaded image: \K.*') - echo "Loaded image: $LOADED_IMAGE" - - # Verify the loaded image exists - docker image inspect "$LOADED_IMAGE" >/dev/null 2>&1 || { - echo "ERROR: Loaded image $LOADED_IMAGE not found after load"; - exit 1; - } - - # Prepare tags - BASE_REPO="${{ parameters.ContainerRegistry }}.azurecr.io/${{ parameters.DeploymentEnvironment }}/$(SetReleaseVariables.DockerImageName)" - VERSIONED_TAG="$BASE_REPO:$(SetReleaseVariables.DockerImageVersion)" - LATEST_TAG="$BASE_REPO:latest" - - echo "Tagging $LOADED_IMAGE as:" - echo " Versioned tag: $VERSIONED_TAG" - echo " Latest tag: $LATEST_TAG" - - # Tag the images - docker tag "$LOADED_IMAGE" "$VERSIONED_TAG" - docker tag "$LOADED_IMAGE" "$LATEST_TAG" - - # Verify tags exist before pushing - docker image inspect "$VERSIONED_TAG" >/dev/null 2>&1 || { - echo "ERROR: Versioned tag $VERSIONED_TAG not found after tagging"; - exit 1; - } - docker image inspect "$LATEST_TAG" >/dev/null 2>&1 || { - echo "ERROR: Latest tag $LATEST_TAG not found after tagging"; - exit 1; - } - - # Push both tags - echo "Pushing versioned tag..."; docker push "$VERSIONED_TAG" - echo "Pushing latest tag..."; docker push "$LATEST_TAG" - echo "Publish complete for ${{ parameters.ServerName }}" - displayName: Load, tag, and push image + - task: PowerShell@2 + displayName: Load, tag, and push Docker images + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-DockerImages.ps1 + arguments: > + -CliName '$(SetReleaseVariables.CliName)' + -Version '$(SetReleaseVariables.DockerImageVersion)' + -BaseRepo '${{ parameters.ContainerRegistry }}.azurecr.io/${{ parameters.DeploymentEnvironment }}/$(SetReleaseVariables.DockerImageName)' + -TarDirectory '$(Pipeline.Workspace)/docker_output' diff --git a/eng/pipelines/templates/jobs/release-anu.yml b/eng/pipelines/templates/jobs/release-anu.yml new file mode 100644 index 0000000000..b310fcc992 --- /dev/null +++ b/eng/pipelines/templates/jobs/release-anu.yml @@ -0,0 +1,23 @@ +# Temporary release pipeline for Docker ARM64 testing +# Only runs Docker release to azuresdkimages.azurecr.io/anu/... +# DELETE THIS FILE after testing is complete + +parameters: +- name: ServerName + type: string +- name: IncludeNative + type: boolean +- name: PackageDocker + type: boolean +- name: PackageVSIX + type: boolean +- name: IsTestPipeline + type: boolean + +jobs: +- ${{ if parameters.PackageDocker }}: + - template: /eng/pipelines/templates/jobs/docker/release-docker.yml + parameters: + ContainerRegistry: 'azuresdkimages' + DeploymentEnvironment: 'anu' # TODO: Revert to 'public' after Docker ARM64 testing + ServerName: ${{ parameters.ServerName }} diff --git a/eng/pipelines/templates/variables/image.yml b/eng/pipelines/templates/variables/image.yml index c9dd2252b5..87ab8d56e8 100644 --- a/eng/pipelines/templates/variables/image.yml +++ b/eng/pipelines/templates/variables/image.yml @@ -3,6 +3,8 @@ variables: - name: LINUXPOOL value: azsdk-pool + - name: LINUXARMPOOL + value: azsdk-pool-arm64 - name: WINDOWSPOOL value: azsdk-pool - name: MACPOOL @@ -10,6 +12,8 @@ variables: - name: LINUXVMIMAGE value: ubuntu-24.04 + - name: LINUXARMVMIMAGE + value: azure-linux-3-arm64 - name: WINDOWSVMIMAGE value: windows-2022 - name: MACVMIMAGE diff --git a/eng/scripts/New-BuildInfo.ps1 b/eng/scripts/New-BuildInfo.ps1 index 10b1852530..11ce722853 100644 --- a/eng/scripts/New-BuildInfo.ps1 +++ b/eng/scripts/New-BuildInfo.ps1 @@ -42,6 +42,14 @@ $additionalPlatforms = @( trimmed = $false specialPurpose = 'docker' } + @{ + name = 'linux-musl-arm64-docker' + operatingSystem = 'linux' + architecture = 'musl-arm64' + native = $false + trimmed = $false + specialPurpose = 'docker' + } ) if ($IncludeNative) { @@ -593,23 +601,55 @@ function Get-ServerMatrix { Write-Host "Forming server matrix" $serverMatrix = [ordered]@{} - $platformName = "linux-musl-x64-docker" + + # MCP Servers that should build ARM64 Docker images in addition to default AMD64 + $arm64DockerServers = @("Azure.Mcp.Server") + + # Docker architecture configurations + $dockerArchConfigs = @( + @{ + Architecture = 'amd64' + PlatformName = 'linux-musl-x64-docker' + Pool = '$(LINUXPOOL)' + VMImage = '$(LINUXVMIMAGE)' + } + @{ + Architecture = 'arm64' + PlatformName = 'linux-musl-arm64-docker' + Pool = '$(LINUXARMPOOL)' + VMImage = '$(LINUXARMVMIMAGE)' + } + ) foreach ($server in $servers) { - $platform = $server.platforms | Where-Object { $_.name -eq $platformName -and -not $_.native } - $executableExtension = $platform.extension $imageName = $server.dockerImageName - if (-not $platform.extension) { $executableExtension = '' } if (-not $server.dockerImageName) { $imageName = "microsoft/" + $server.cliName + "-mcp" } - $serverMatrix[$server.name] = [ordered]@{ - ServerName = $server.name - CliName = $server.cliName - ArtifactPath = $server.artifactPath - Platform = $platformName - Version = $server.version - ImageName = $imageName - ExecutableName = $server.cliName + $executableExtension - DockerLocalTag = $imageName + ":" + $BuildId + + foreach ($archConfig in $dockerArchConfigs) { + if ($archConfig.Architecture -eq 'arm64' -and $arm64DockerServers -notcontains $server.name) { + # Skip ARM64 for MCP servers not in the opt-in list + continue + } + + $platform = $server.platforms | Where-Object { $_.name -eq $archConfig.PlatformName -and -not $_.native } + $executableExtension = $platform.extension ?? '' + + $matrixKey = "$($server.name)_$($archConfig.Architecture)" + + $serverMatrix[$matrixKey] = [ordered]@{ + ServerName = $server.name + CliName = $server.cliName + ArtifactPath = $server.artifactPath + Version = $server.version + ImageName = $imageName + ExecutableName = $server.cliName + $executableExtension + DockerLocalTag = $imageName + ":" + $BuildId + # Docker build configuration + Platform = $archConfig.PlatformName + Architecture = $archConfig.Architecture + Pool = $archConfig.Pool + VMImage = $archConfig.VMImage + } } } diff --git a/eng/scripts/Publish-DockerImages.ps1 b/eng/scripts/Publish-DockerImages.ps1 new file mode 100644 index 0000000000..4b7fadebc4 --- /dev/null +++ b/eng/scripts/Publish-DockerImages.ps1 @@ -0,0 +1,259 @@ +<# +.SYNOPSIS + Publishes Docker images to a container registry with multi-architecture manifest support. + +.DESCRIPTION + This script loads Docker images from tar files, tags them with architecture suffixes, + pushes them to a container registry, and creates multi-arch manifests when multiple + architectures are present. + +.PARAMETER CliName + The CLI name used to identify tar files (e.g., 'azmcp' matches 'azmcp-amd64-image.tar'). + +.PARAMETER Version + The version tag for the Docker images (e.g., '2.0.0'). + +.PARAMETER BaseRepo + The base repository URL without tag (e.g., 'azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp'). + +.PARAMETER TarDirectory + The directory containing the Docker image tar files. + +.EXAMPLE + ./Publish-DockerImages.ps1 -CliName 'azmcp' -Version '2.0.0' -BaseRepo 'azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp' -TarDirectory './docker_output' +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$CliName, + + [Parameter(Mandatory = $true)] + [string]$Version, + + [Parameter(Mandatory = $true)] + [string]$BaseRepo, + + [Parameter(Mandatory = $true)] + [string]$TarDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Invoke-DockerCommand { + param( + [string[]]$Arguments, + [switch]$CaptureOutput + ) + + Write-Host "docker $($Arguments -join ' ')" -ForegroundColor DarkGray + + if ($CaptureOutput) { + $output = & docker @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Error "Docker command failed with exit code $LASTEXITCODE`: $output" + } + return $output + } + else { + & docker @Arguments + if ($LASTEXITCODE -ne 0) { + Write-Error "Docker command failed with exit code $LASTEXITCODE" + } + } +} + +# E.g., When publishing "Azure.Mcp.Server", given TarDirectory "docker_output" and +# CliName "azmcp", finds docker tar files: azmcp-amd64-image.tar, azmcp-arm64-image.tar +function Get-DockerTarFiles { + param( + [string]$TarDirectory, + [string]$CliName + ) + + Write-Host "Discovering tar files..." + $tarPattern = Join-Path $TarDirectory "$CliName-*-image.tar" + $tarFiles = Get-ChildItem -Path $tarPattern -ErrorAction SilentlyContinue + + if (-not $tarFiles -or $tarFiles.Count -eq 0) { + Write-Host "ERROR: No tar files found matching pattern $tarPattern" -ForegroundColor Red + Write-Host "Directory contents:" + Get-ChildItem -Path $TarDirectory | ForEach-Object { Write-Host " $_" } + exit 1 + } + + Write-Host "Found tar files:" + $tarFiles | ForEach-Object { Write-Host " $($_.FullName)" } + + return $tarFiles +} + +# E.g., Given TarPath "docker_output/azmcp-arm64-image.tar" and CliName "azmcp", +# returns the architecture suffix "arm64" +function Get-ArchitectureFromTarFile { + param([string]$TarPath, [string]$CliName) + + $fileName = [System.IO.Path]::GetFileName($TarPath) + # Using the pattern '{CliName}-{arch}-image.tar' capture {arch} + $pattern = "^$([regex]::Escape($CliName))-(.+)-image\.tar$" + if ($fileName -match $pattern) { + return $Matches[1] + } + Write-Error "Could not extract architecture from tar file name: $fileName" +} + +# Loads Docker image from tar file and returns the image name. +# E.g., Given TarPath "docker_output/azmcp-arm64-image.tar", loads the image and +# returns the image name "azure-sdk/azure-mcp:99999" (DockerLocalTag from build stage) +function Load-DockerImage { + param([string]$TarPath) + + Write-Host "Loading $TarPath..." + $output = Invoke-DockerCommand -Arguments @('load', '-i', $TarPath) -CaptureOutput + + # Parse "Loaded image: " from output + foreach ($line in $output) { + if ($line -match 'Loaded image:\s*(.+)$') { + $imageName = $Matches[1].Trim() + Write-Host "Loaded image: $imageName" + return $imageName + } + } + + Write-Error "Could not parse loaded image name from docker load output: $output" +} + +# Checks if an image exists in the local Docker daemon. +function Test-DockerImageExists { + param([string]$ImageName) + + $null = docker image inspect $ImageName 2>&1 + return $LASTEXITCODE -eq 0 +} + +# Loads a Docker image from tar, verifies it exists, and tags it with architecture suffix. +# E.g., for "Azure.Mcp.Server", given TarPath "docker_output/azmcp-arm64-image.tar" +# creates tag: azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp:2.0.0-arm64 +# Returns a hashtable with ArchTag, LoadedImage, and Architecture. +function Import-ArchitectureImage { + param( + [string]$TarPath, + [string]$CliName, + [string]$BaseRepo, + [string]$Version + ) + + $arch = Get-ArchitectureFromTarFile -TarPath $TarPath -CliName $CliName + Write-Host "Processing $arch" -ForegroundColor Yellow + + $localImage = Load-DockerImage -TarPath $TarPath + + if (-not (Test-DockerImageExists -ImageName $localImage)) { + Write-Error "Loaded image '$localImage' not found after load" + } + + $archTag = "${BaseRepo}:${Version}-${arch}" + Write-Host "Tagging as: $archTag" + Invoke-DockerCommand -Arguments @('tag', $localImage, $archTag) + + Write-Host "" + + return @{ + # E.g., azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp:2.0.0-arm64 + ArchTag = $archTag + # E.g., azure-sdk/azure-mcp:99999 + LocalImage = $localImage + # E.g., arm64 + Architecture = $arch + } +} + +# Tags and pushes a single-arch image to the registry. +# E.g., for "Fabric.Mcp.Server", given LocalImage "fabric/fabric-mcp:99999", +# tags and pushes as "azuresdkimages.azurecr.io/public/fabric/fabric-mcp:2.0.0" +function Publish-SingleArchImage { + param( + [string]$LocalImage, + [string]$TargetTag + ) + + Write-Host "Tagging $LocalImage as $TargetTag..." + Invoke-DockerCommand -Arguments @('tag', $LocalImage, $TargetTag) + Write-Host "Pushing $TargetTag..." + Invoke-DockerCommand -Arguments @('push', $TargetTag) +} + +function New-MultiArchManifest { + param( + [string]$ManifestTag, + [string[]]$ArchTags + ) + + Write-Host "Creating multi-arch manifest for $ManifestTag..." + Invoke-DockerCommand -Arguments (@('manifest', 'create', $ManifestTag) + $ArchTags) + Invoke-DockerCommand -Arguments @('manifest', 'push', $ManifestTag) +} + +# Main +Write-Host "Docker Release" -ForegroundColor Cyan +Write-Host "CLI Name: $CliName" +Write-Host "Version: $Version" +Write-Host "Base Repo: $BaseRepo" +Write-Host "Tar Directory: $TarDirectory" + +# Discover Docker image tar files +Write-Host "" +$tarFiles = Get-DockerTarFiles -TarDirectory $TarDirectory -CliName $CliName + +# Load and tag each Docker image tar file +$imageInfos = @() +foreach ($tar in $tarFiles) { + $info = Import-ArchitectureImage -TarPath $tar.FullName -CliName $CliName -BaseRepo $BaseRepo -Version $Version + $imageInfos += $info +} + +# Create and push tags +Write-Host "Publishing tags" -ForegroundColor Cyan + +# E.g., azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp:2.0.0 +$versionedTag = "${BaseRepo}:${Version}" +# E.g., azuresdkimages.azurecr.io/public/azure-sdk/azure-mcp:latest +$latestTag = "${BaseRepo}:latest" + +if ($imageInfos.Count -gt 1) { + # Multi-arch: push arch-specific tags, then create manifests + # + $archTags = @() + foreach ($info in $imageInfos) { + Write-Host "Pushing $($info.ArchTag)..." + Invoke-DockerCommand -Arguments @('push', $info.ArchTag) + $archTags += $info.ArchTag + } + + Write-Host "" + New-MultiArchManifest -ManifestTag $versionedTag -ArchTags $archTags + New-MultiArchManifest -ManifestTag $latestTag -ArchTags $archTags + + Write-Host "Publish complete" -ForegroundColor Green + Write-Host "Published tags:" + Write-Host " - $versionedTag (manifest)" + Write-Host " - $latestTag (manifest)" + foreach ($tag in $archTags) { + Write-Host " - $tag" + } +} +else { + # Single arch: just tag and push versioned/latest directly (no arch-specific tag published) + # + Write-Host "Single architecture - publishing directly..." + $localImage = $imageInfos[0].LocalImage + + Publish-SingleArchImage -LocalImage $localImage -TargetTag $versionedTag + Publish-SingleArchImage -LocalImage $localImage -TargetTag $latestTag + + Write-Host "Publish complete" -ForegroundColor Green + Write-Host "Published tags:" + Write-Host " - $versionedTag" + Write-Host " - $latestTag" +}