From 741d82b83bc650986b3b2ce2d40ae4f12e4be3bd Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Mon, 11 May 2026 13:33:40 +0100 Subject: [PATCH 1/5] ci: deploy to a dedicated vnet with subnet This alters the deploy to deploy to a dedicated Azure vnet and subnet, which have a dedicated IP attached. These are created outside the CI/CD pipeline, so the IP remains fixed for as long as possible. --- .github/workflows/build-and-deploy-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-deploy-job.yml b/.github/workflows/build-and-deploy-job.yml index dd0cc05..f16022d 100644 --- a/.github/workflows/build-and-deploy-job.yml +++ b/.github/workflows/build-and-deploy-job.yml @@ -32,6 +32,7 @@ jobs: echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV} echo "RESOURCE_GROUP_BASE_NAME=rg-${APP_NAME}" >> ${GITHUB_ENV} echo "STORAGE_ACCOUNT_NAME=sa${APP_NAME//-/}$TARGET_ENVIRONMENT" >> ${GITHUB_ENV} + echo "APP_NAME=${APP_NAME}" >> ${GITHUB_ENV} - name: 'Print calculated environment variables' run: | @@ -39,7 +40,7 @@ jobs: echo $CONTAINER_INSTANCE_BASE_NAME echo $RESOURCE_GROUP_BASE_NAME echo $STORAGE_ACCOUNT_NAME - + echo $APP_NAME - name: 'Checkout GitHub Action' uses: actions/checkout@v4 @@ -137,6 +138,8 @@ jobs: az -v az container create --debug \ --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ + --vnet "${{ env.APP_NAME }}-${{ env.TARGET_ENVIRONMENT }}-vnet" \ + --subnet "${{ env.APP_NAME }}-${{ env.TARGET_ENVIRONMENT }}-subnet" \ --file ./azure-deployment/azure-resource-manager-deployment-manifest.yml - name: 'Re-generate the website links' From 0cbfaedc314dbe190958057758fd21ad52e9b2bd Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Mon, 11 May 2026 13:55:25 +0100 Subject: [PATCH 2/5] feat: script to create vnets for public IP --- azure-provision/create-vnets-public-ips.sh | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 azure-provision/create-vnets-public-ips.sh diff --git a/azure-provision/create-vnets-public-ips.sh b/azure-provision/create-vnets-public-ips.sh new file mode 100755 index 0000000..70212d0 --- /dev/null +++ b/azure-provision/create-vnets-public-ips.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +# This script creates the virtual networks, subnets and public IPs for the bulk data service. + +RESOURCE_GROUP_NAME="rg-bulk-data-service-vnets" +LOCATION="uksouth" + +az group create --name "$RESOURCE_GROUP_NAME" --location "$LOCATION" + +for ENV in dev prod; do + az network vnet create --resource-group "$RESOURCE_GROUP_NAME" \ + --name "bulk-data-service-${ENV}-vnet" \ + --address-prefix 10.0.0.0/16 \ + --subnet-name "bulk-data-service-${ENV}-subnet" \ + --subnet-prefix 10.0.1.0/24 + + az network vnet subnet update --resource-group "$RESOURCE_GROUP_NAME" \ + --vnet-name "bulk-data-service-${ENV}-vnet" \ + --name "bulk-data-service-${ENV}-subnet" \ + --delegation Microsoft.ContainerInstance/containerGroups + + az network public-ip create \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --name "bulk-data-service-${ENV}-public-ip" \ + --sku Standard \ + --allocation-method Static \ + --location "$LOCATION" + + az network nat gateway create \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --name "bulk-data-service-${ENV}-nat-gateway" \ + --location "$LOCATION" \ + --public-ip-addresses "bulk-data-service-${ENV}-public-ip" \ + --idle-timeout 10 + + az network vnet subnet update \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --vnet-name "bulk-data-service-${ENV}-vnet" \ + --name "bulk-data-service-${ENV}-subnet" \ + --nat-gateway "bulk-data-service-${ENV}-nat-gateway" +done From 680329cfa4c2f8dc0cf4070e09c634d3bf8cb7af Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Mon, 11 May 2026 15:41:01 +0100 Subject: [PATCH 3/5] fix(ci): add missing env var for MQ --- .github/workflows/build-and-deploy-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-deploy-job.yml b/.github/workflows/build-and-deploy-job.yml index f16022d..40897da 100644 --- a/.github/workflows/build-and-deploy-job.yml +++ b/.github/workflows/build-and-deploy-job.yml @@ -106,6 +106,7 @@ jobs: LOG_WORKSPACE_KEY: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'LOG_WORKSPACE_KEY')] }} # Variables which configure the app + AZURE_SERVICE_BUS_DATASET_CHECK_RESULTS_TOPIC_NAME: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_DATASET_CHECK_RESULTS_TOPIC_NAME')] }} AZURE_SERVICE_BUS_REGISTRY_SUB_NAME: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_REGISTRY_SUB_NAME')] }} AZURE_SERVICE_BUS_REGISTRY_TOPIC_NAME: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_REGISTRY_TOPIC_NAME')] }} AZURE_SERVICE_BUS_WAIT_TIME: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_WAIT_TIME')] }} From 180eb3bc576ea298685c7c37104c2010d998062f Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Mon, 11 May 2026 14:21:58 +0100 Subject: [PATCH 4/5] build: bump version number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4a485dd..867b89a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bulk-data-service" -version = "1.4.5" +version = "1.4.6" requires-python = ">= 3.12.6" readme = "README.md" dependencies = [ From cdad6ad98db3681a119b7ece821a491aebd27e8d Mon Sep 17 00:00:00 2001 From: Simon K <6615834+simon-20@users.noreply.github.com> Date: Mon, 11 May 2026 13:55:57 +0100 Subject: [PATCH 5/5] docs: update README, CHANGELOG - vnet setup Also adds a missing env var for the MQ setup. --- CHANGELOG.md | 10 ++++++++++ README.md | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c89b3fe..41df74e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Removed +## [1.4.6] - 2026-05-11 + +### Changed + +- Updated deploy to use dedicated vnet & subnet + +### Fixed + +- Added env var for the MQ topic name to the GitHub workflow. + ## [1.4.5] - 2026-05-06 ### Changed diff --git a/README.md b/README.md index 6da6280..fedeecb 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,8 @@ pytest-watcher . ### Initial Provisioning +#### Bulk Data Service App + You can create an Azure-based instance of Bulk Data Service using the `azure-create-resources.sh` script. It must be run from the root of the repository, and it requires (i) the environment variable `BDS_DB_ADMIN_PASSWORD` to be set with the password for the database, and (ii) a single parameter which is the name of the environment/instance. For instance, the following command will create a dev instance: ```bash @@ -249,6 +251,16 @@ This will create a resource group on Azure called `rg-bulk-data-service-dev`, an At the end of its run, the `azure-create-resources.sh` script will print out various secrets which need to be added to Github Actions. +**NOTE**: This is only really useful for temporary deployment or initial setup; once you're setup with CI/CD, the GitHub action does all this. + +#### Bulk Data Service Network and Public IP + +The Bulk Data Service is deployed to a dedicated vnet with subnet and attached NAT Gateway which has a public IP. To ensure the IP remains, these are not destroyed and re-created on every release (like the Azure Container Instances are). To create the networks and public IPs for dev and production, run: + +```bash +./azure-provision/create-vnets-public-ips.sh +``` + ### Deployment - Versioning The app version is set in `pyproject.toml`, and this is read by the app to use in the `User-Agent` header. When making a new release, set the version here to the appropriate value. Then, when releasing the app using the normal IATI Python app deployment process, choose the tag name to match the version chosen.