From 9969439aa013cd0863887aa6bfc52e5adfdbb642 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Mar 2026 15:02:07 -0800 Subject: [PATCH 1/7] git sync working but can be improved --- airflow/helm/values.tmpl.yaml | 22 +++++++++++++++++++++- airflow/helm/values_high_load.tmpl.yaml | 13 +++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index 1ed89de1..f4a8d90c 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -292,6 +292,12 @@ config: enable_proxy_fix: 'True' dags: + # Disable built-in git-sync (doesn't work with existingClaim) + # We'll add it manually as extraContainers below + gitSync: + enabled: false + + # Keep persistence enabled for OGC API dynamic DAG deployment persistence: # Enable persistent volume for storing dags enabled: true @@ -317,6 +323,20 @@ dagProcessor: - key: "karpenter.k8s.aws/instance-cpu" operator: "In" values: ["4"] + extraContainers: + - name: git-sync + image: registry.k8s.io/git-sync/git-sync:v4.2.4 + args: + - --repo=https://github.com/MAAP-Project/airflow-dags.git + - --branch=main + - --root=/git + - --dest=repo + - --period=60s + - --max-failures=0 + - --link=current + volumeMounts: + - name: dags + mountPath: /git env: - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" @@ -335,7 +355,7 @@ env: # https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/security/api.html extraEnv: | - name: AIRFLOW__CORE__DAGS_FOLDER - value: "/opt/airflow/dags" + value: "/opt/airflow/dags/repo" - name: AIRFLOW__CORE__PLUGINS_FOLDER value: "/opt/airflow/plugins" - name: AIRFLOW__CORE__LAZY_LOAD_PLUGINS diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 2b62148a..077c30fb 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -291,6 +291,19 @@ config: enable_proxy_fix: 'True' dags: + # Git-sync configuration for automatic DAG synchronization from GitHub + gitSync: + enabled: true + repo: https://github.com/MAAP-Project/airflow-dags.git + branch: main + ref: main + rev: HEAD + depth: 1 + maxFailures: 0 + subPath: "/opt/airflow/dags" # All DAGs are at repository root level + period: 60s # Sync every 60 seconds + + # Keep persistence enabled for OGC API dynamic DAG deployment persistence: # Enable persistent volume for storing dags enabled: true From ba97b6411b014a9bc2eb5609992af0e52cb80c9c Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 4 Mar 2026 16:24:31 -0800 Subject: [PATCH 2/7] git sync now correctly working --- airflow/helm/values.tmpl.yaml | 13 +++--------- airflow/helm/values_high_load.tmpl.yaml | 28 ++++++++++++------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index f4a8d90c..6bc0c850 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -292,12 +292,6 @@ config: enable_proxy_fix: 'True' dags: - # Disable built-in git-sync (doesn't work with existingClaim) - # We'll add it manually as extraContainers below - gitSync: - enabled: false - - # Keep persistence enabled for OGC API dynamic DAG deployment persistence: # Enable persistent volume for storing dags enabled: true @@ -325,15 +319,14 @@ dagProcessor: values: ["4"] extraContainers: - name: git-sync - image: registry.k8s.io/git-sync/git-sync:v4.2.4 + image: registry.k8s.io/git-sync/git-sync:v4.6.0 args: - --repo=https://github.com/MAAP-Project/airflow-dags.git - - --branch=main + - --ref=main - --root=/git - - --dest=repo + - --link=repo - --period=60s - --max-failures=0 - - --link=current volumeMounts: - name: dags mountPath: /git diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 077c30fb..18a40069 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -291,19 +291,6 @@ config: enable_proxy_fix: 'True' dags: - # Git-sync configuration for automatic DAG synchronization from GitHub - gitSync: - enabled: true - repo: https://github.com/MAAP-Project/airflow-dags.git - branch: main - ref: main - rev: HEAD - depth: 1 - maxFailures: 0 - subPath: "/opt/airflow/dags" # All DAGs are at repository root level - period: 60s # Sync every 60 seconds - - # Keep persistence enabled for OGC API dynamic DAG deployment persistence: # Enable persistent volume for storing dags enabled: true @@ -330,6 +317,19 @@ dagProcessor: - key: "karpenter.k8s.aws/instance-cpu" operator: "In" values: [ "16", "32", "64" ] # Scheduler might benefit from higher CPU + extraContainers: + - name: git-sync + image: registry.k8s.io/git-sync/git-sync:v4.6.0 + args: + - --repo=https://github.com/MAAP-Project/airflow-dags.git + - --ref=main + - --root=/git + - --link=repo + - --period=60s + - --max-failures=0 + volumeMounts: + - name: dags + mountPath: /git env: - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" @@ -348,7 +348,7 @@ env: # https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/security/api.html extraEnv: | - name: AIRFLOW__CORE__DAGS_FOLDER - value: "/opt/airflow/dags" + value: "/opt/airflow/dags/repo" - name: AIRFLOW__CORE__PLUGINS_FOLDER value: "/opt/airflow/plugins" - name: AIRFLOW__CORE__LAZY_LOAD_PLUGINS From 8415ef09d8cf96e45136cfda182fba2325e40622 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 11 Mar 2026 10:48:42 -0700 Subject: [PATCH 3/7] attempting to read from multiple DAG locations from s3 --- airflow/docker/multi-git-sync/Dockerfile | 29 ++ airflow/docker/multi-git-sync/README.md | 160 +++++++++ .../multi-git-sync/dag_repos_airflow.json | 14 + airflow/docker/multi-git-sync/sync-repos.py | 315 ++++++++++++++++++ terraform-unity/.terraform.lock.hcl | 26 +- .../data.tf | 2 + .../locals.tf | 5 +- .../main.tf | 135 +++++++- .../variables.tf | 2 +- terraform-unity/provider.tf | 4 + terraform-unity/variables.tf | 8 +- 11 files changed, 664 insertions(+), 36 deletions(-) create mode 100644 airflow/docker/multi-git-sync/Dockerfile create mode 100644 airflow/docker/multi-git-sync/README.md create mode 100644 airflow/docker/multi-git-sync/dag_repos_airflow.json create mode 100644 airflow/docker/multi-git-sync/sync-repos.py diff --git a/airflow/docker/multi-git-sync/Dockerfile b/airflow/docker/multi-git-sync/Dockerfile new file mode 100644 index 00000000..14b07b8c --- /dev/null +++ b/airflow/docker/multi-git-sync/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.11-slim + +# Install git and other required tools +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +RUN pip install --no-cache-dir boto3 + +# Create gitsync user (non-root) +RUN useradd -m -u 1000 -s /bin/bash gitsync && \ + mkdir -p /dag-catalog/repos /dag-catalog/current && \ + chown -R gitsync:gitsync /dag-catalog + +# Copy sync script +COPY sync-repos.py /usr/local/bin/sync-repos.py +RUN chmod +x /usr/local/bin/sync-repos.py + +# Switch to non-root user +USER gitsync + +# Set working directory +WORKDIR /dag-catalog + +# Set entrypoint +ENTRYPOINT ["/usr/local/bin/sync-repos.py"] diff --git a/airflow/docker/multi-git-sync/README.md b/airflow/docker/multi-git-sync/README.md new file mode 100644 index 00000000..5acb5062 --- /dev/null +++ b/airflow/docker/multi-git-sync/README.md @@ -0,0 +1,160 @@ +# Multi-Git-Sync Container + +A custom container that syncs multiple git repositories based on configuration stored in AWS S3. + +## Features + +- **Dynamic repository configuration**: Read repository list from S3 +- **Automatic polling**: Checks for configuration changes every 60 seconds (configurable) +- **Multi-repo support**: Syncs multiple repositories to separate subdirectories +- **No restart required**: Automatically picks up new repositories without pod restart +- **IRSA support**: Uses IAM Roles for Service Accounts (IRSA) for AWS authentication + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `S3_BUCKET` | S3 bucket containing repo configuration file | `unity-dev-sps-config-smce` | +| `S3_KEY` | S3 object key for repo configuration file | `dag_repos_airflow.json` | +| `AWS_REGION` | AWS region for S3 | `us-west-2` | +| `SYNC_ROOT` | Root directory for synced repositories | `/dag-catalog` | +| `POLL_INTERVAL` | Polling interval in seconds | `60` | + +## S3 Configuration File Format + +The S3 object dag_repos_airflow.json should contain a JSON array of repository configurations: + +```json +[ + { + "url": "https://github.com/unity-sds/unity-sps.git", + "ref": "main", + "path": "airflow/dags", + "name": "unity-sps" + }, + { + "url": "https://github.com/org/another-repo.git", + "ref": "develop", + "path": "dags", + "name": "another-repo" + } +] +``` + +### Configuration Fields + +- `url`: Git repository URL (HTTPS) +- `ref`: Git ref to checkout (branch, tag, or commit) +- `path`: Subdirectory within the repo to expose (relative path) +- `name`: Unique name for the repository (used as directory name) + +## Directory Structure + +``` +/dag-catalog/ +├── repos/ +│ ├── unity-sps/ # Git clone of unity-sps repo +│ └── another-repo/ # Git clone of another-repo +└── current/ + ├── unity-sps -> ../repos/unity-sps/airflow/dags + └── another-repo -> ../repos/another-repo/dags +``` + +## IAM Permissions Required + +The container requires the following IAM permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": "arn:aws:s3:::unity-*-sps-config-smce/*" + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": "arn:aws:s3:::unity-*-sps-config-smce" + } + ] +} +``` + +## Usage in Kubernetes + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: ogc-processes-api +spec: + serviceAccountName: ogc-processes-api # Must have IRSA annotation + containers: + - name: multi-git-sync + image: ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 + env: + - name: S3_BUCKET + value: "unity-dev-sps-config-smce" + - name: S3_KEY + value: "dag_repos_airflow.json" + - name: AWS_REGION + value: "us-west-2" + - name: SYNC_ROOT + value: "/dag-catalog" + - name: POLL_INTERVAL + value: "60" + volumeMounts: + - name: dag-catalog + mountPath: /dag-catalog + volumes: + - name: dag-catalog + emptyDir: {} +``` + +## Building the Image + +```bash +docker build -t ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 . +docker push ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 +``` + +## Adding a New Repository + +To add a new repository without restarting the pod: + +```bash +# Update the S3 configuration file +aws s3 cp s3://unity-dev-sps-config-smce/dag_repos_airflow.json - | \ + jq '. + [{"url": "https://github.com/org/new-repo.git", "ref": "main", "path": "dags", "name": "new-repo"}]' | \ + aws s3 cp - s3://unity-dev-sps-config-smce/dag_repos_airflow.json + +# Wait 60-120 seconds for the next poll cycle +# Check logs to verify sync +kubectl logs -c multi-git-sync --tail=50 +``` + +## Troubleshooting + +### Check container logs +```bash +kubectl logs -c multi-git-sync -f +``` + +### Verify S3 configuration file +```bash +aws s3 cp s3://unity-dev-sps-config-smce/dag_repos_airflow.json - | jq . +``` + +### Check synced directories +```bash +kubectl exec -c multi-git-sync -- ls -la /dag-catalog/current/ +``` + +### Common Issues + +1. **"S3 object not found"**: Ensure the S3 object exists at the specified bucket and key, and the IAM role has access +2. **"S3 bucket not found"**: Verify the bucket name is correct and exists in the AWS account +3. **"Failed to clone repository"**: Check git URL is correct and accessible (public repo or credentials configured) +4. **"Source path does not exist"**: Verify the `path` field in the config points to a valid directory in the repo diff --git a/airflow/docker/multi-git-sync/dag_repos_airflow.json b/airflow/docker/multi-git-sync/dag_repos_airflow.json new file mode 100644 index 00000000..f36e08d2 --- /dev/null +++ b/airflow/docker/multi-git-sync/dag_repos_airflow.json @@ -0,0 +1,14 @@ +[ + { + "url": "https://github.com/MAAP-Project/airflow-dags.git", + "ref": "main", + "path": ".", + "name": "MAAP DAGs" + }, + { + "url": "https://github.com/grallewellyn/unity-dags-2.git", + "ref": "main", + "path": ".", + "name": "Project 2" + } +] \ No newline at end of file diff --git a/airflow/docker/multi-git-sync/sync-repos.py b/airflow/docker/multi-git-sync/sync-repos.py new file mode 100644 index 00000000..0cfd1527 --- /dev/null +++ b/airflow/docker/multi-git-sync/sync-repos.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +""" +Multi-repository git sync script that polls AWS S3. + +This script: +1. Reads repository configurations from an S3 object +2. Clones or updates each repository +3. Creates symlinks for each repo under /dag-catalog/current/{repo-name} +4. Polls S3 every POLL_INTERVAL seconds for configuration changes +5. Automatically picks up new repositories without restart +""" + +import json +import logging +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + +import boto3 +from botocore.exceptions import BotoErr, ClientError + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + stream=sys.stdout +) +logger = logging.getLogger('multi-git-sync') + +# Environment variables +S3_BUCKET = os.environ.get('S3_BUCKET', 'unity-dev-sps-config-smce') +S3_KEY = os.environ.get('S3_KEY', 'dag_repos_airflow.json') +AWS_REGION = os.environ.get('AWS_REGION', 'us-west-2') +SYNC_ROOT = os.environ.get('SYNC_ROOT', '/dag-catalog') +POLL_INTERVAL = int(os.environ.get('POLL_INTERVAL', '60')) + +# Directory paths +REPOS_DIR = Path(SYNC_ROOT) / 'repos' +CURRENT_DIR = Path(SYNC_ROOT) / 'current' + + +class RepoConfig: + """Repository configuration.""" + + def __init__(self, url: str, ref: str, path: str, name: str): + self.url = url + self.ref = ref + self.path = path + self.name = name + + def __eq__(self, other): + if not isinstance(other, RepoConfig): + return False + return (self.url == other.url and + self.ref == other.ref and + self.path == other.path and + self.name == other.name) + + def __hash__(self): + return hash((self.url, self.ref, self.path, self.name)) + + @classmethod + def from_dict(cls, data: Dict) -> 'RepoConfig': + """Create RepoConfig from dictionary.""" + return cls( + url=data['url'], + ref=data['ref'], + path=data['path'], + name=data['name'] + ) + + +class GitSyncManager: + """Manages syncing of multiple git repositories.""" + + def __init__(self): + self.s3_client = boto3.client('s3', region_name=AWS_REGION) + self.current_repos: Dict[str, RepoConfig] = {} + + # Ensure directories exist + REPOS_DIR.mkdir(parents=True, exist_ok=True) + CURRENT_DIR.mkdir(parents=True, exist_ok=True) + + logger.info(f"Initialized GitSyncManager") + logger.info(f" S3 Bucket: {S3_BUCKET}") + logger.info(f" S3 Key: {S3_KEY}") + logger.info(f" AWS Region: {AWS_REGION}") + logger.info(f" Sync Root: {SYNC_ROOT}") + logger.info(f" Poll Interval: {POLL_INTERVAL}s") + + def get_repo_configs(self) -> Optional[List[RepoConfig]]: + """Fetch repository configurations from S3.""" + try: + response = self.s3_client.get_object(Bucket=S3_BUCKET, Key=S3_KEY) + content = response['Body'].read().decode('utf-8') + + # Parse JSON + repos_data = json.loads(content) + + if not isinstance(repos_data, list): + logger.error(f"S3 object content is not a list: {type(repos_data)}") + return None + + configs = [] + for repo_data in repos_data: + try: + config = RepoConfig.from_dict(repo_data) + configs.append(config) + except (KeyError, TypeError) as e: + logger.error(f"Invalid repo configuration: {repo_data}, error: {e}") + continue + + logger.debug(f"Fetched {len(configs)} repository configurations from S3") + return configs + + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == 'NoSuchKey': + logger.error(f"S3 object not found: s3://{S3_BUCKET}/{S3_KEY}") + elif error_code == 'NoSuchBucket': + logger.error(f"S3 bucket not found: {S3_BUCKET}") + else: + logger.error(f"AWS error fetching S3 object: {e}") + return None + except json.JSONDecodeError as e: + logger.error(f"Failed to parse S3 object as JSON: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error fetching repo configs: {e}") + return None + + def run_command(self, cmd: List[str], cwd: Optional[Path] = None) -> bool: + """Run a shell command and return success status.""" + try: + result = subprocess.run( + cmd, + cwd=cwd, + check=True, + capture_output=True, + text=True + ) + logger.debug(f"Command succeeded: {' '.join(cmd)}") + if result.stdout: + logger.debug(f" stdout: {result.stdout.strip()}") + return True + except subprocess.CalledProcessError as e: + logger.error(f"Command failed: {' '.join(cmd)}") + logger.error(f" Exit code: {e.returncode}") + if e.stdout: + logger.error(f" stdout: {e.stdout.strip()}") + if e.stderr: + logger.error(f" stderr: {e.stderr.strip()}") + return False + except Exception as e: + logger.error(f"Unexpected error running command {' '.join(cmd)}: {e}") + return False + + def clone_repo(self, config: RepoConfig) -> bool: + """Clone a repository.""" + repo_dir = REPOS_DIR / config.name + + logger.info(f"Cloning repository: {config.name}") + logger.info(f" URL: {config.url}") + logger.info(f" Ref: {config.ref}") + logger.info(f" Path: {config.path}") + + # Clone the repository + if not self.run_command(['git', 'clone', config.url, str(repo_dir)]): + return False + + # Checkout the specified ref + if not self.run_command(['git', 'checkout', config.ref], cwd=repo_dir): + return False + + logger.info(f"Successfully cloned: {config.name}") + return True + + def update_repo(self, config: RepoConfig) -> bool: + """Update an existing repository.""" + repo_dir = REPOS_DIR / config.name + + logger.info(f"Updating repository: {config.name}") + + # Fetch latest changes + if not self.run_command(['git', 'fetch', 'origin'], cwd=repo_dir): + logger.warning(f"Failed to fetch updates for {config.name}, continuing...") + return False + + # Checkout the specified ref + if not self.run_command(['git', 'checkout', config.ref], cwd=repo_dir): + return False + + # Pull if it's a branch + if not self.run_command(['git', 'pull'], cwd=repo_dir): + logger.warning(f"Failed to pull updates for {config.name}, may not be on a branch") + + logger.info(f"Successfully updated: {config.name}") + return True + + def create_symlink(self, config: RepoConfig) -> bool: + """Create symlink from current/{name} to repos/{name}/{path}.""" + source = REPOS_DIR / config.name / config.path + target = CURRENT_DIR / config.name + + # Remove existing symlink if it exists + if target.exists() or target.is_symlink(): + target.unlink() + + # Verify source exists + if not source.exists(): + logger.error(f"Source path does not exist: {source}") + return False + + # Create symlink + try: + target.symlink_to(source) + logger.info(f"Created symlink: {target} -> {source}") + return True + except Exception as e: + logger.error(f"Failed to create symlink {target} -> {source}: {e}") + return False + + def sync_repo(self, config: RepoConfig) -> bool: + """Sync a single repository (clone or update).""" + repo_dir = REPOS_DIR / config.name + + try: + # Clone if doesn't exist, otherwise update + if not repo_dir.exists(): + if not self.clone_repo(config): + return False + else: + if not self.update_repo(config): + return False + + # Create/update symlink + return self.create_symlink(config) + + except Exception as e: + logger.error(f"Error syncing repo {config.name}: {e}") + return False + + def remove_stale_repos(self, configs: List[RepoConfig]): + """Remove symlinks for repos no longer in configuration.""" + current_names = {config.name for config in configs} + + # Check each symlink in current/ + for symlink in CURRENT_DIR.iterdir(): + if symlink.name not in current_names: + logger.info(f"Removing stale repository symlink: {symlink.name}") + try: + symlink.unlink() + except Exception as e: + logger.error(f"Failed to remove symlink {symlink}: {e}") + + def sync_all_repos(self): + """Fetch configs from S3 and sync all repositories.""" + logger.info("Starting sync cycle...") + + # Fetch repository configurations + configs = self.get_repo_configs() + if configs is None: + logger.error("Failed to fetch repository configurations, skipping sync cycle") + return + + if not configs: + logger.warning("No repositories configured in S3") + return + + logger.info(f"Found {len(configs)} repositories to sync") + + # Sync each repository + success_count = 0 + for config in configs: + if self.sync_repo(config): + success_count += 1 + self.current_repos[config.name] = config + + # Remove stale repositories + self.remove_stale_repos(configs) + + logger.info(f"Sync cycle complete: {success_count}/{len(configs)} repositories synced successfully") + + def run(self): + """Main loop: poll S3 and sync repositories.""" + logger.info("Starting multi-git-sync service...") + + # Initial sync + self.sync_all_repos() + + # Poll loop + while True: + try: + time.sleep(POLL_INTERVAL) + self.sync_all_repos() + except KeyboardInterrupt: + logger.info("Received interrupt signal, shutting down...") + break + except Exception as e: + logger.error(f"Unexpected error in main loop: {e}") + # Continue running despite errors + time.sleep(POLL_INTERVAL) + + +def main(): + """Entry point.""" + manager = GitSyncManager() + manager.run() + + +if __name__ == '__main__': + main() diff --git a/terraform-unity/.terraform.lock.hcl b/terraform-unity/.terraform.lock.hcl index 61004c96..73d85a5d 100644 --- a/terraform-unity/.terraform.lock.hcl +++ b/terraform-unity/.terraform.lock.hcl @@ -89,22 +89,22 @@ provider "registry.terraform.io/hashicorp/kubernetes" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.6.1" + version = "2.7.0" constraints = ">= 2.5.1" hashes = [ - "h1:DbiR/D2CPigzCGweYIyJH0N0x04oyI5xiZ9wSW/s3kQ=", - "zh:10050d08f416de42a857e4b6f76809aae63ea4ec6f5c852a126a915dede814b4", - "zh:2df2a3ebe9830d4759c59b51702e209fe053f47453cb4688f43c063bac8746b7", - "zh:2e759568bcc38c86ca0e43701d34cf29945736fdc8e429c5b287ddc2703c7b18", - "zh:6a62a34e48500ab4aea778e355e162ebde03260b7a9eb9edc7e534c84fbca4c6", - "zh:74373728ba32a1d5450a3a88ac45624579e32755b086cd4e51e88d9aca240ef6", + "h1:sSwlfp2etjCaE9hIF7bJBDjRIhDCVFglEOVyiCI7vgs=", + "zh:261fec71bca13e0a7812dc0d8ae9af2b4326b24d9b2e9beab3d2400fab5c5f9a", + "zh:308da3b5376a9ede815042deec5af1050ec96a5a5410a2206ae847d82070a23e", + "zh:3d056924c420464dc8aba10e1915956b2e5c4d55b11ffff79aa8be563fbfe298", + "zh:643256547b155459c45e0a3e8aab0570db59923c68daf2086be63c444c8c445b", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8dddae588971a996f622e7589cd8b9da7834c744ac12bfb59c97fa77ded95255", - "zh:946f82f66353bb97aefa8d95c4ca86db227f9b7c50b82415289ac47e4e74d08d", - "zh:e9a5c09e6f35e510acf15b666fd0b34a30164cecdcd81ce7cda0f4b2dade8d91", - "zh:eafe5b873ef42b32feb2f969c38ff8652507e695620cbaf03b9db714bee52249", - "zh:ec146289fa27650c9d433bb5c7847379180c0b7a323b1b94e6e7ad5d2a7dbe71", - "zh:fc882c35ce05631d76c0973b35adde26980778fc81d9da81a2fade2b9d73423b", + "zh:7aa4d0b853f84205e8cf79f30c9b2c562afbfa63592f7231b6637e5d7a6b5b27", + "zh:7dc251bbc487d58a6ab7f5b07ec9edc630edb45d89b761dba28e0e2ba6b1c11f", + "zh:7ee0ca546cd065030039168d780a15cbbf1765a4c70cd56d394734ab112c93da", + "zh:b1d5d80abb1906e6c6b3685a52a0192b4ca6525fe090881c64ec6f67794b1300", + "zh:d81ea9856d61db3148a4fc6c375bf387a721d78fc1fea7a8823a027272a47a78", + "zh:df0a1f0afc947b8bfc88617c1ad07a689ce3bd1a29fd97318392e6bdd32b230b", + "zh:dfbcad800240e0c68c43e0866f2a751cff09777375ec701918881acf67a268da", ] } diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf index 5bb04943..63c155e3 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf @@ -72,6 +72,8 @@ data "aws_security_groups" "venue_proxy_sg" { data "aws_region" "current" {} +data "aws_caller_identity" "current" {} + data "aws_ssm_parameter" "unity_client_id" { name = "/sps/processing/workflows/unity_client_id" } diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/locals.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/locals.tf index 8402c200..cd0cd8a3 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/locals.tf @@ -1,6 +1,8 @@ locals { - resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) + resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) + s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"])) + dag_catalog_config_bucket = "mdps-airflow-${var.venue}-dag-sources" common_tags = { Name = "" Venue = var.venue @@ -15,4 +17,5 @@ locals { } load_balancer_port = 5001 region = data.aws_region.current.name + oidc_provider_url = replace(data.aws_eks_cluster.cluster.identity[0].oidc[0].issuer, "https://", "") } diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 69ee66a0..dc4f2f3b 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -1,3 +1,107 @@ +# IAM Policy for S3 access +resource "aws_iam_policy" "ogc_processes_api_s3_policy" { + name = "${var.project}-${var.venue}-ogc-api-s3-policy" + description = "Allows OGC Processes API to read DAG catalog repository configuration from S3" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject" + ] + Resource = "arn:aws:s3:::${local.dag_catalog_config_bucket}/*" + }, + { + Effect = "Allow" + Action = [ + "s3:ListBucket" + ] + Resource = "arn:aws:s3:::${local.dag_catalog_config_bucket}" + } + ] + }) + + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "s3-policy") + Component = "OGC" + Stack = "OGC" + }) +} + +# IAM Role for IRSA (IAM Roles for Service Accounts) +resource "aws_iam_role" "ogc_processes_api_role" { + name = "${var.project}-${var.venue}-ogc-api-role" + description = "IAM role for OGC Processes API pod to access AWS resources via IRSA" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider_url}" + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${local.oidc_provider_url}:sub" = "system:serviceaccount:${var.kubernetes_namespace}:ogc-processes-api" + "${local.oidc_provider_url}:aud" = "sts.amazonaws.com" + } + } + } + ] + }) + + managed_policy_arns = [aws_iam_policy.ogc_processes_api_s3_policy.arn] + permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/zsmce-tenantOperator-AMI-APIG" + + tags = merge(local.common_tags, { + Name = format(local.resource_name_prefix, "iam-role") + Component = "OGC" + Stack = "OGC" + }) +} + +# Kubernetes Service Account with IRSA annotation +resource "kubernetes_service_account" "ogc_processes_api" { + metadata { + name = "ogc-processes-api" + namespace = data.kubernetes_namespace.service_area.metadata[0].name + annotations = { + "eks.amazonaws.com/role-arn" = aws_iam_role.ogc_processes_api_role.arn + } + } +} + +# S3 Object for repository list (bucket is managed externally) +# NOTE: This resource is commented out because the file is managed manually +# The application will still read from this S3 location via the multi-git-sync container +# resource "aws_s3_object" "dag_catalog_repos" { +# bucket = local.dag_catalog_config_bucket +# key = "dag_repos_airflow.json" +# content = jsonencode([ +# { +# url = var.dag_catalog_repo.url +# ref = var.dag_catalog_repo.ref +# path = var.dag_catalog_repo.dags_directory_path +# name = "unity-sps" +# } +# ]) +# content_type = "application/json" +# +# tags = merge(local.common_tags, { +# Name = format(local.resource_name_prefix, "dag-repos-config") +# Component = "OGC" +# Stack = "OGC" +# }) +# +# lifecycle { +# ignore_changes = [content] +# } +# } + resource "kubernetes_deployment" "redis" { metadata { name = "ogc-processes-api-redis-lock" @@ -94,6 +198,7 @@ resource "kubernetes_deployment" "ogc_processes_api" { } } spec { + service_account_name = kubernetes_service_account.ogc_processes_api.metadata[0].name affinity { node_affinity { required_during_scheduling_ignored_during_execution { @@ -155,7 +260,7 @@ resource "kubernetes_deployment" "ogc_processes_api" { } env { name = "DAG_CATALOG_DIRECTORY" - value = "/dag-catalog/current/${var.dag_catalog_repo.dags_directory_path}" + value = "/dag-catalog/current/" } env { name = "DEPLOYED_DAGS_DIRECTORY" @@ -171,31 +276,27 @@ resource "kubernetes_deployment" "ogc_processes_api" { } } container { - name = "git-sync" - image = "${var.docker_images.git_sync.name}:${var.docker_images.git_sync.tag}" + name = "multi-git-sync" + image = "${var.docker_images.multi_git_sync.name}:${var.docker_images.multi_git_sync.tag}" env { - name = "GITSYNC_REPO" - value = var.dag_catalog_repo.url + name = "S3_BUCKET" + value = local.dag_catalog_config_bucket } env { - name = "GITSYNC_REF" - value = var.dag_catalog_repo.ref + name = "S3_KEY" + value = "dag_repos_airflow.json" } env { - name = "GITSYNC_ROOT" - value = "/dag-catalog" - } - env { - name = "GITSYNC_LINK" - value = "current" + name = "AWS_REGION" + value = data.aws_region.current.name } env { - name = "GITSYNC_PERIOD" - value = "3s" + name = "SYNC_ROOT" + value = "/dag-catalog" } env { - name = "GITSYNC_ONE_TIME" - value = "false" + name = "POLL_INTERVAL" + value = "60" } volume_mount { name = "dag-catalog" diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf index 7509c8ba..b4e74849 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf @@ -55,7 +55,7 @@ variable "docker_images" { name = string tag = string }) - git_sync = object({ + multi_git_sync = object({ name = string tag = string }) diff --git a/terraform-unity/provider.tf b/terraform-unity/provider.tf index 68fac698..c8791851 100644 --- a/terraform-unity/provider.tf +++ b/terraform-unity/provider.tf @@ -1,3 +1,7 @@ +provider "aws" { + region = "us-west-2" +} + provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint token = data.aws_eks_cluster_auth.cluster.token diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 9d590a7c..decc6843 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -91,7 +91,7 @@ variable "ogc_processes_docker_images" { name = string tag = string }) - git_sync = object({ + multi_git_sync = object({ name = string tag = string }) @@ -105,9 +105,9 @@ variable "ogc_processes_docker_images" { name = "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api" tag = "2.1.0" } - git_sync = { - name = "registry.k8s.io/git-sync/git-sync" - tag = "v4.2.4" + multi_git_sync = { + name = "ghcr.io/unity-sds/unity-sps/multi-git-sync" + tag = "1.0.0" }, redis = { name = "redis" From 3ebdee9d3966d5bf818c9ffd06cc2357960e92ee Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 11 Mar 2026 14:00:21 -0700 Subject: [PATCH 4/7] updated image name and permissions --- airflow/docker/multi-git-sync/Dockerfile | 3 ++- airflow/docker/multi-git-sync/README.md | 6 +++--- terraform-unity/variables.tf | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airflow/docker/multi-git-sync/Dockerfile b/airflow/docker/multi-git-sync/Dockerfile index 14b07b8c..943006cd 100644 --- a/airflow/docker/multi-git-sync/Dockerfile +++ b/airflow/docker/multi-git-sync/Dockerfile @@ -17,7 +17,8 @@ RUN useradd -m -u 1000 -s /bin/bash gitsync && \ # Copy sync script COPY sync-repos.py /usr/local/bin/sync-repos.py -RUN chmod +x /usr/local/bin/sync-repos.py +RUN chmod 755 /usr/local/bin/sync-repos.py && \ + chown gitsync:gitsync /usr/local/bin/sync-repos.py # Switch to non-root user USER gitsync diff --git a/airflow/docker/multi-git-sync/README.md b/airflow/docker/multi-git-sync/README.md index 5acb5062..fc19a5b4 100644 --- a/airflow/docker/multi-git-sync/README.md +++ b/airflow/docker/multi-git-sync/README.md @@ -93,7 +93,7 @@ spec: serviceAccountName: ogc-processes-api # Must have IRSA annotation containers: - name: multi-git-sync - image: ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 + image: jplmdps:v1.0.0 env: - name: S3_BUCKET value: "unity-dev-sps-config-smce" @@ -116,8 +116,8 @@ spec: ## Building the Image ```bash -docker build -t ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 . -docker push ghcr.io/unity-sds/unity-sps/multi-git-sync:1.0.0 +docker build -t jplmdps/multi-git-sync:v1.0.0 . +docker push jplmdps/multi-git-sync:v1.0.0 ``` ## Adding a New Repository diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index decc6843..0b425700 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -106,8 +106,8 @@ variable "ogc_processes_docker_images" { tag = "2.1.0" } multi_git_sync = { - name = "ghcr.io/unity-sds/unity-sps/multi-git-sync" - tag = "1.0.0" + name = "jplmdps/multi-git-sync" + tag = "v1.0.0" }, redis = { name = "redis" From 90472924baceee9ae7dc7936f987a9a4483508c8 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 17 Mar 2026 11:51:12 -0700 Subject: [PATCH 5/7] deployment working for both repos and updating in airflow --- .../multi-git-sync/dag_repos_airflow.json | 4 +- airflow/docker/multi-git-sync/sync-repos.py | 14 ++++--- airflow/helm/values.tmpl.yaml | 26 +++++++----- airflow/helm/values_high_load.tmpl.yaml | 26 +++++++----- terraform-unity/main.tf | 30 +++++++------- .../terraform-unity-sps-airflow/locals.tf | 1 + .../terraform-unity-sps-airflow/main.tf | 41 ++++++++++--------- .../terraform-unity-sps-airflow/variables.tf | 8 ++++ .../main.tf | 2 +- .../variables.tf | 12 ++++-- terraform-unity/variables.tf | 20 +++++---- 11 files changed, 109 insertions(+), 75 deletions(-) diff --git a/airflow/docker/multi-git-sync/dag_repos_airflow.json b/airflow/docker/multi-git-sync/dag_repos_airflow.json index f36e08d2..fd36e37e 100644 --- a/airflow/docker/multi-git-sync/dag_repos_airflow.json +++ b/airflow/docker/multi-git-sync/dag_repos_airflow.json @@ -3,12 +3,12 @@ "url": "https://github.com/MAAP-Project/airflow-dags.git", "ref": "main", "path": ".", - "name": "MAAP DAGs" + "name": "MAAP_DAGs" }, { "url": "https://github.com/grallewellyn/unity-dags-2.git", "ref": "main", "path": ".", - "name": "Project 2" + "name": "Project_2" } ] \ No newline at end of file diff --git a/airflow/docker/multi-git-sync/sync-repos.py b/airflow/docker/multi-git-sync/sync-repos.py index 0cfd1527..c4d1cc34 100644 --- a/airflow/docker/multi-git-sync/sync-repos.py +++ b/airflow/docker/multi-git-sync/sync-repos.py @@ -20,7 +20,7 @@ from typing import Dict, List, Optional import boto3 -from botocore.exceptions import BotoErr, ClientError +from botocore.exceptions import ClientError # Configure logging logging.basicConfig( @@ -214,13 +214,17 @@ def create_symlink(self, config: RepoConfig) -> bool: logger.error(f"Source path does not exist: {source}") return False - # Create symlink + # Create relative symlink so it works regardless of mount point + # From /dag-catalog/current/{name} to /dag-catalog/repos/{name}/{path} + # Relative path: ../repos/{name}/{path} + relative_source = Path('..') / 'repos' / config.name / config.path + try: - target.symlink_to(source) - logger.info(f"Created symlink: {target} -> {source}") + target.symlink_to(relative_source) + logger.info(f"Created relative symlink: {target} -> {relative_source} (absolute: {source})") return True except Exception as e: - logger.error(f"Failed to create symlink {target} -> {source}: {e}") + logger.error(f"Failed to create symlink {target} -> {relative_source}: {e}") return False def sync_repo(self, config: RepoConfig) -> bool: diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index 6bc0c850..c6acd549 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -318,18 +318,22 @@ dagProcessor: operator: "In" values: ["4"] extraContainers: - - name: git-sync - image: registry.k8s.io/git-sync/git-sync:v4.6.0 - args: - - --repo=https://github.com/MAAP-Project/airflow-dags.git - - --ref=main - - --root=/git - - --link=repo - - --period=60s - - --max-failures=0 + - name: multi-git-sync + image: ${multi_git_sync_image_repo}:${multi_git_sync_image_tag} + env: + - name: S3_BUCKET + value: ${dag_catalog_config_bucket} + - name: S3_KEY + value: dag_repos_airflow.json + - name: AWS_REGION + value: us-west-2 + - name: SYNC_ROOT + value: /dag-catalog + - name: POLL_INTERVAL + value: "60" volumeMounts: - name: dags - mountPath: /git + mountPath: /dag-catalog env: - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" @@ -348,7 +352,7 @@ env: # https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/security/api.html extraEnv: | - name: AIRFLOW__CORE__DAGS_FOLDER - value: "/opt/airflow/dags/repo" + value: "/opt/airflow/dags/current" - name: AIRFLOW__CORE__PLUGINS_FOLDER value: "/opt/airflow/plugins" - name: AIRFLOW__CORE__LAZY_LOAD_PLUGINS diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 18a40069..3cd1eecd 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -318,18 +318,22 @@ dagProcessor: operator: "In" values: [ "16", "32", "64" ] # Scheduler might benefit from higher CPU extraContainers: - - name: git-sync - image: registry.k8s.io/git-sync/git-sync:v4.6.0 - args: - - --repo=https://github.com/MAAP-Project/airflow-dags.git - - --ref=main - - --root=/git - - --link=repo - - --period=60s - - --max-failures=0 + - name: multi-git-sync + image: ${multi_git_sync_image_repo}:${multi_git_sync_image_tag} + env: + - name: S3_BUCKET + value: ${dag_catalog_config_bucket} + - name: S3_KEY + value: dag_repos_airflow.json + - name: AWS_REGION + value: us-west-2 + - name: SYNC_ROOT + value: /dag-catalog + - name: POLL_INTERVAL + value: "60" volumeMounts: - name: dags - mountPath: /git + mountPath: /dag-catalog env: - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" @@ -348,7 +352,7 @@ env: # https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/security/api.html extraEnv: | - name: AIRFLOW__CORE__DAGS_FOLDER - value: "/opt/airflow/dags/repo" + value: "/opt/airflow/dags/current" - name: AIRFLOW__CORE__PLUGINS_FOLDER value: "/opt/airflow/plugins" - name: AIRFLOW__CORE__LAZY_LOAD_PLUGINS diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf index ce780675..c1324e44 100644 --- a/terraform-unity/main.tf +++ b/terraform-unity/main.tf @@ -102,26 +102,28 @@ module "unity-sps-airflow" { airflow_webserver_username = var.airflow_webserver_username airflow_webserver_password = var.airflow_webserver_password docker_images = var.airflow_docker_images + multi_git_sync_docker_image = var.multi_git_sync_docker_image helm_charts = var.helm_charts helm_values_template = var.helm_values_template karpenter_node_pools = module.unity-sps-karpenter-node-config.karpenter_node_pools } module "unity-sps-ogc-processes-api" { - source = "./modules/terraform-unity-sps-ogc-processes-api" - project = var.project - venue = var.venue - service_area = var.service_area - release = var.release - kubernetes_namespace = kubernetes_namespace.service_area.metadata[0].name - db_instance_identifier = module.unity-sps-database.db_instance_identifier - db_secret_arn = module.unity-sps-database.db_secret_arn - airflow_deployed_dags_pvc = module.unity-sps-airflow.airflow_deployed_dags_pvc - airflow_webserver_username = var.airflow_webserver_username - airflow_webserver_password = var.airflow_webserver_password - docker_images = var.ogc_processes_docker_images - dag_catalog_repo = var.dag_catalog_repo - karpenter_node_pools = module.unity-sps-karpenter-node-config.karpenter_node_pools + source = "./modules/terraform-unity-sps-ogc-processes-api" + project = var.project + venue = var.venue + service_area = var.service_area + release = var.release + kubernetes_namespace = kubernetes_namespace.service_area.metadata[0].name + db_instance_identifier = module.unity-sps-database.db_instance_identifier + db_secret_arn = module.unity-sps-database.db_secret_arn + airflow_deployed_dags_pvc = module.unity-sps-airflow.airflow_deployed_dags_pvc + airflow_webserver_username = var.airflow_webserver_username + airflow_webserver_password = var.airflow_webserver_password + docker_images = var.ogc_processes_docker_images + multi_git_sync_docker_image = var.multi_git_sync_docker_image + dag_catalog_repo = var.dag_catalog_repo + karpenter_node_pools = module.unity-sps-karpenter-node-config.karpenter_node_pools } module "unity-sps-initiators" { diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf index a9bfa754..837ed703 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf @@ -2,6 +2,7 @@ locals { resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"])) + dag_catalog_config_bucket = "mdps-airflow-${var.venue}-dag-sources" common_tags = { Name = "" Venue = var.venue diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index 1ef830de..a4664a1f 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -412,25 +412,28 @@ resource "helm_release" "airflow" { namespace = data.kubernetes_namespace.service_area.metadata[0].name values = [ templatefile("${path.module}/../../../airflow/helm/${var.helm_values_template}", { - airflow_image_repo = var.docker_images.airflow.name - airflow_image_tag = var.docker_images.airflow.tag - kubernetes_namespace = data.kubernetes_namespace.service_area.metadata[0].name - metadata_secret_name = local.airflow_metadata_kubernetes_secret - webserver_secret_name = local.airflow_webserver_kubernetes_secret - airflow_logs_s3_location = "s3://${aws_s3_bucket.airflow_logs.id}" - airflow_worker_role_arn = aws_iam_role.airflow_worker_role.arn - workers_pvc_name = kubernetes_persistent_volume_claim.airflow_kpo.metadata[0].name - dags_pvc_name = kubernetes_persistent_volume_claim.airflow_deployed_dags.metadata[0].name - webserver_instance_name = format(local.resource_name_prefix, "airflow") - webserver_navbar_color = local.airflow_webserver_navbar_color - service_area = upper(var.service_area) - service_area_version = var.release - unity_project = var.project - unity_venue = var.venue - unity_cluster_name = data.aws_eks_cluster.cluster.name - karpenter_node_pools = join(",", var.karpenter_node_pools) - cwl_dag_ecr_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-west-2.amazonaws.com" - airflow_base_url = local.airflow_base_url + airflow_image_repo = var.docker_images.airflow.name + airflow_image_tag = var.docker_images.airflow.tag + multi_git_sync_image_repo = var.multi_git_sync_docker_image.name + multi_git_sync_image_tag = var.multi_git_sync_docker_image.tag + dag_catalog_config_bucket = local.dag_catalog_config_bucket + kubernetes_namespace = data.kubernetes_namespace.service_area.metadata[0].name + metadata_secret_name = local.airflow_metadata_kubernetes_secret + webserver_secret_name = local.airflow_webserver_kubernetes_secret + airflow_logs_s3_location = "s3://${aws_s3_bucket.airflow_logs.id}" + airflow_worker_role_arn = aws_iam_role.airflow_worker_role.arn + workers_pvc_name = kubernetes_persistent_volume_claim.airflow_kpo.metadata[0].name + dags_pvc_name = kubernetes_persistent_volume_claim.airflow_deployed_dags.metadata[0].name + webserver_instance_name = format(local.resource_name_prefix, "airflow") + webserver_navbar_color = local.airflow_webserver_navbar_color + service_area = upper(var.service_area) + service_area_version = var.release + unity_project = var.project + unity_venue = var.venue + unity_cluster_name = data.aws_eks_cluster.cluster.name + karpenter_node_pools = join(",", var.karpenter_node_pools) + cwl_dag_ecr_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-west-2.amazonaws.com" + airflow_base_url = local.airflow_base_url # Keycloak Direct OIDC authentication configuration webserver_config = indent(4, templatefile("${path.module}/../../../airflow/config/webserver_config.py.tpl", { keycloak_role_mapping = var.keycloak_role_mapping diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf index 99d0aec5..089177e5 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf @@ -77,6 +77,14 @@ variable "docker_images" { }) } +variable "multi_git_sync_docker_image" { + description = "Docker image for multi-git-sync container." + type = object({ + name = string + tag = string + }) +} + variable "karpenter_node_pools" { description = "Names of the Karpenter node pools" type = list(string) diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index dc4f2f3b..8513c221 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -277,7 +277,7 @@ resource "kubernetes_deployment" "ogc_processes_api" { } container { name = "multi-git-sync" - image = "${var.docker_images.multi_git_sync.name}:${var.docker_images.multi_git_sync.tag}" + image = "${var.multi_git_sync_docker_image.name}:${var.multi_git_sync_docker_image.tag}" env { name = "S3_BUCKET" value = local.dag_catalog_config_bucket diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf index b4e74849..6e61b52c 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf @@ -55,10 +55,6 @@ variable "docker_images" { name = string tag = string }) - multi_git_sync = object({ - name = string - tag = string - }) redis = object({ name = string tag = string @@ -66,6 +62,14 @@ variable "docker_images" { }) } +variable "multi_git_sync_docker_image" { + description = "Docker image for multi-git-sync container." + type = object({ + name = string + tag = string + }) +} + variable "dag_catalog_repo" { description = "Git repository that stores the catalog of Airflow DAGs." type = object({ diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 0b425700..28c1dc1a 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -68,6 +68,18 @@ variable "helm_values_template" { default = "values.tmpl.yaml" } +variable "multi_git_sync_docker_image" { + description = "Docker image for multi-git-sync container (shared by Airflow and OGC Processes API)." + type = object({ + name = string + tag = string + }) + default = { + name = "jplmdps/multi-git-sync" + tag = "develop3" + } +} + variable "airflow_docker_images" { description = "Docker images for the associated Airflow services." type = object({ @@ -91,10 +103,6 @@ variable "ogc_processes_docker_images" { name = string tag = string }) - multi_git_sync = object({ - name = string - tag = string - }) redis = object({ name = string tag = string @@ -105,10 +113,6 @@ variable "ogc_processes_docker_images" { name = "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api" tag = "2.1.0" } - multi_git_sync = { - name = "jplmdps/multi-git-sync" - tag = "v1.0.0" - }, redis = { name = "redis" tag = "7.4.0" From 903d300067985d2b1718f6df28ee64f2d58fb2db Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Tue, 17 Mar 2026 15:14:00 -0700 Subject: [PATCH 6/7] refactored and updated comments- only bug is changing the path --- README.md | 10 ++- airflow/docker/multi-git-sync/README.md | 74 +------------------ airflow/helm/values.tmpl.yaml | 2 +- airflow/helm/values_high_load.tmpl.yaml | 2 +- terraform-unity/main.tf | 32 ++++---- .../terraform-unity-sps-airflow/data.tf | 2 + .../terraform-unity-sps-airflow/main.tf | 1 + .../main.tf | 27 ------- terraform-unity/provider.tf | 4 - terraform-unity/variables.tf | 2 +- 10 files changed, 34 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index c28d20fb..eafc01e5 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,15 @@ This guide provides a quick way to get started with our project. Please see our ### Build Instructions (if applicable) -N/A +When building in a new AWS account or for a new venue, need to create the s3 bucket `mdps-airflow-{venue}-dag-sources` that contains the JSON dag_repos_airflow.json to specify where DAGs should be read from. Schema example is: +``` +{ + "url": "https://github.com/MAAP-Project/airflow-dags.git", + "ref": "main", (branch) + "path": ".", (don't need to include repo name) + "name": "MAAP_DAGs" (needs to be unique across other entries) + }, +``` ### Test Instructions (if applicable) diff --git a/airflow/docker/multi-git-sync/README.md b/airflow/docker/multi-git-sync/README.md index fc19a5b4..cf2b1067 100644 --- a/airflow/docker/multi-git-sync/README.md +++ b/airflow/docker/multi-git-sync/README.md @@ -7,7 +7,7 @@ A custom container that syncs multiple git repositories based on configuration s - **Dynamic repository configuration**: Read repository list from S3 - **Automatic polling**: Checks for configuration changes every 60 seconds (configurable) - **Multi-repo support**: Syncs multiple repositories to separate subdirectories -- **No restart required**: Automatically picks up new repositories without pod restart +- **No restart required**: Automatically picks up new repositories without pod restart, just edit the s3 file - **IRSA support**: Uses IAM Roles for Service Accounts (IRSA) for AWS authentication ## Environment Variables @@ -28,8 +28,8 @@ The S3 object dag_repos_airflow.json should contain a JSON array of repository c [ { "url": "https://github.com/unity-sds/unity-sps.git", - "ref": "main", - "path": "airflow/dags", + "ref": "main", (branch) + "path": "airflow/dags", (dont need to include repo name and "." for root) "name": "unity-sps" }, { @@ -60,59 +60,6 @@ The S3 object dag_repos_airflow.json should contain a JSON array of repository c └── another-repo -> ../repos/another-repo/dags ``` -## IAM Permissions Required - -The container requires the following IAM permissions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:GetObject"], - "Resource": "arn:aws:s3:::unity-*-sps-config-smce/*" - }, - { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": "arn:aws:s3:::unity-*-sps-config-smce" - } - ] -} -``` - -## Usage in Kubernetes - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: ogc-processes-api -spec: - serviceAccountName: ogc-processes-api # Must have IRSA annotation - containers: - - name: multi-git-sync - image: jplmdps:v1.0.0 - env: - - name: S3_BUCKET - value: "unity-dev-sps-config-smce" - - name: S3_KEY - value: "dag_repos_airflow.json" - - name: AWS_REGION - value: "us-west-2" - - name: SYNC_ROOT - value: "/dag-catalog" - - name: POLL_INTERVAL - value: "60" - volumeMounts: - - name: dag-catalog - mountPath: /dag-catalog - volumes: - - name: dag-catalog - emptyDir: {} -``` - ## Building the Image ```bash @@ -120,21 +67,6 @@ docker build -t jplmdps/multi-git-sync:v1.0.0 . docker push jplmdps/multi-git-sync:v1.0.0 ``` -## Adding a New Repository - -To add a new repository without restarting the pod: - -```bash -# Update the S3 configuration file -aws s3 cp s3://unity-dev-sps-config-smce/dag_repos_airflow.json - | \ - jq '. + [{"url": "https://github.com/org/new-repo.git", "ref": "main", "path": "dags", "name": "new-repo"}]' | \ - aws s3 cp - s3://unity-dev-sps-config-smce/dag_repos_airflow.json - -# Wait 60-120 seconds for the next poll cycle -# Check logs to verify sync -kubectl logs -c multi-git-sync --tail=50 -``` - ## Troubleshooting ### Check container logs diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index c6acd549..0932a1ee 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -326,7 +326,7 @@ dagProcessor: - name: S3_KEY value: dag_repos_airflow.json - name: AWS_REGION - value: us-west-2 + value: ${aws_region} - name: SYNC_ROOT value: /dag-catalog - name: POLL_INTERVAL diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 3cd1eecd..1ee3b8e0 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -326,7 +326,7 @@ dagProcessor: - name: S3_KEY value: dag_repos_airflow.json - name: AWS_REGION - value: us-west-2 + value: ${aws_region} - name: SYNC_ROOT value: /dag-catalog - name: POLL_INTERVAL diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf index c1324e44..d2e64487 100644 --- a/terraform-unity/main.tf +++ b/terraform-unity/main.tf @@ -89,23 +89,23 @@ module "unity-sps-karpenter-node-config" { } module "unity-sps-airflow" { - source = "./modules/terraform-unity-sps-airflow" - project = var.project - venue = var.venue - service_area = var.service_area - release = var.release - kubeconfig_filepath = var.kubeconfig_filepath - kubernetes_namespace = kubernetes_namespace.service_area.metadata[0].name - db_instance_identifier = module.unity-sps-database.db_instance_identifier - db_secret_arn = module.unity-sps-database.db_secret_arn - efs_file_system_id = module.unity-sps-efs.file_system_id - airflow_webserver_username = var.airflow_webserver_username - airflow_webserver_password = var.airflow_webserver_password - docker_images = var.airflow_docker_images + source = "./modules/terraform-unity-sps-airflow" + project = var.project + venue = var.venue + service_area = var.service_area + release = var.release + kubeconfig_filepath = var.kubeconfig_filepath + kubernetes_namespace = kubernetes_namespace.service_area.metadata[0].name + db_instance_identifier = module.unity-sps-database.db_instance_identifier + db_secret_arn = module.unity-sps-database.db_secret_arn + efs_file_system_id = module.unity-sps-efs.file_system_id + airflow_webserver_username = var.airflow_webserver_username + airflow_webserver_password = var.airflow_webserver_password + docker_images = var.airflow_docker_images multi_git_sync_docker_image = var.multi_git_sync_docker_image - helm_charts = var.helm_charts - helm_values_template = var.helm_values_template - karpenter_node_pools = module.unity-sps-karpenter-node-config.karpenter_node_pools + helm_charts = var.helm_charts + helm_values_template = var.helm_values_template + karpenter_node_pools = module.unity-sps-karpenter-node-config.karpenter_node_pools } module "unity-sps-ogc-processes-api" { diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf index a4bec754..12373e60 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf @@ -1,5 +1,7 @@ data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + data "aws_eks_cluster" "cluster" { name = format(local.resource_name_prefix, "eks") } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index a4664a1f..3bdf2acb 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -432,6 +432,7 @@ resource "helm_release" "airflow" { unity_venue = var.venue unity_cluster_name = data.aws_eks_cluster.cluster.name karpenter_node_pools = join(",", var.karpenter_node_pools) + aws_region = data.aws_region.current.name cwl_dag_ecr_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-west-2.amazonaws.com" airflow_base_url = local.airflow_base_url # Keycloak Direct OIDC authentication configuration diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 8513c221..60b32820 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -75,33 +75,6 @@ resource "kubernetes_service_account" "ogc_processes_api" { } } -# S3 Object for repository list (bucket is managed externally) -# NOTE: This resource is commented out because the file is managed manually -# The application will still read from this S3 location via the multi-git-sync container -# resource "aws_s3_object" "dag_catalog_repos" { -# bucket = local.dag_catalog_config_bucket -# key = "dag_repos_airflow.json" -# content = jsonencode([ -# { -# url = var.dag_catalog_repo.url -# ref = var.dag_catalog_repo.ref -# path = var.dag_catalog_repo.dags_directory_path -# name = "unity-sps" -# } -# ]) -# content_type = "application/json" -# -# tags = merge(local.common_tags, { -# Name = format(local.resource_name_prefix, "dag-repos-config") -# Component = "OGC" -# Stack = "OGC" -# }) -# -# lifecycle { -# ignore_changes = [content] -# } -# } - resource "kubernetes_deployment" "redis" { metadata { name = "ogc-processes-api-redis-lock" diff --git a/terraform-unity/provider.tf b/terraform-unity/provider.tf index c8791851..68fac698 100644 --- a/terraform-unity/provider.tf +++ b/terraform-unity/provider.tf @@ -1,7 +1,3 @@ -provider "aws" { - region = "us-west-2" -} - provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint token = data.aws_eks_cluster_auth.cluster.token diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 28c1dc1a..7e273e62 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -76,7 +76,7 @@ variable "multi_git_sync_docker_image" { }) default = { name = "jplmdps/multi-git-sync" - tag = "develop3" + tag = "v1.0.0" } } From 165df3e076856347511dfddcd43b7a2d9f80a108 Mon Sep 17 00:00:00 2001 From: grallewellyn Date: Wed, 18 Mar 2026 14:45:21 -0700 Subject: [PATCH 7/7] creating a single folder name from a path --- .../multi-git-sync/dag_repos_airflow.json | 2 +- airflow/docker/multi-git-sync/sync-repos.py | 39 ++++++++++++++++--- airflow/helm/values.tmpl.yaml | 2 + airflow/helm/values_high_load.tmpl.yaml | 2 + 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/airflow/docker/multi-git-sync/dag_repos_airflow.json b/airflow/docker/multi-git-sync/dag_repos_airflow.json index fd36e37e..17a7a811 100644 --- a/airflow/docker/multi-git-sync/dag_repos_airflow.json +++ b/airflow/docker/multi-git-sync/dag_repos_airflow.json @@ -8,7 +8,7 @@ { "url": "https://github.com/grallewellyn/unity-dags-2.git", "ref": "main", - "path": ".", + "path": "folder1/folder2", "name": "Project_2" } ] \ No newline at end of file diff --git a/airflow/docker/multi-git-sync/sync-repos.py b/airflow/docker/multi-git-sync/sync-repos.py index c4d1cc34..60062ba6 100644 --- a/airflow/docker/multi-git-sync/sync-repos.py +++ b/airflow/docker/multi-git-sync/sync-repos.py @@ -91,6 +91,28 @@ def __init__(self): logger.info(f" Sync Root: {SYNC_ROOT}") logger.info(f" Poll Interval: {POLL_INTERVAL}s") + def get_symlink_name(self, config: RepoConfig) -> str: + """ + Generate symlink name including path component. + This ensures that changing the path creates a new symlink, + preventing stale DAGs from remaining visible in Airflow. + + Examples: + - name="Project_2", path="." -> "Project_2" + - name="Project_2", path="folder1/folder2" -> "Project_2__folder1_folder2" + """ + if config.path == ".": + # Root path - just use the name + return config.name + + # Sanitize path for use in filename + # Replace slashes with double underscores, keep only alphanumeric and underscores + sanitized_path = config.path.replace("/", "_").replace("\\", "_") + sanitized_path = "".join(c for c in sanitized_path if c.isalnum() or c == "_") + sanitized_path = sanitized_path.strip("_") + + return f"{config.name}__{sanitized_path}" + def get_repo_configs(self) -> Optional[List[RepoConfig]]: """Fetch repository configurations from S3.""" try: @@ -201,9 +223,10 @@ def update_repo(self, config: RepoConfig) -> bool: return True def create_symlink(self, config: RepoConfig) -> bool: - """Create symlink from current/{name} to repos/{name}/{path}.""" + """Create symlink from current/{name}__{path} to repos/{name}/{path}.""" source = REPOS_DIR / config.name / config.path - target = CURRENT_DIR / config.name + symlink_name = self.get_symlink_name(config) + target = CURRENT_DIR / symlink_name # Remove existing symlink if it exists if target.exists() or target.is_symlink(): @@ -215,7 +238,7 @@ def create_symlink(self, config: RepoConfig) -> bool: return False # Create relative symlink so it works regardless of mount point - # From /dag-catalog/current/{name} to /dag-catalog/repos/{name}/{path} + # From /dag-catalog/current/{name}__{path} to /dag-catalog/repos/{name}/{path} # Relative path: ../repos/{name}/{path} relative_source = Path('..') / 'repos' / config.name / config.path @@ -248,12 +271,16 @@ def sync_repo(self, config: RepoConfig) -> bool: return False def remove_stale_repos(self, configs: List[RepoConfig]): - """Remove symlinks for repos no longer in configuration.""" - current_names = {config.name for config in configs} + """ + Remove symlinks for repos no longer in configuration or with changed paths. + This ensures that changing a repo's path will remove the old symlink. + """ + # Build set of expected symlink names + expected_symlink_names = {self.get_symlink_name(config) for config in configs} # Check each symlink in current/ for symlink in CURRENT_DIR.iterdir(): - if symlink.name not in current_names: + if symlink.name not in expected_symlink_names: logger.info(f"Removing stale repository symlink: {symlink.name}") try: symlink.unlink() diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index 0932a1ee..6ed545c7 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -379,6 +379,8 @@ extraEnv: | value: "10" - name: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL value: "5" + - name: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL + value: "60" - name: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT value: "1024" - name: AIRFLOW__WEBSERVER__EXPOSE_CONFIG diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 1ee3b8e0..91bb3cd0 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -385,6 +385,8 @@ extraEnv: | value: "10" - name: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL value: "0" + - name: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL + value: "60" - name: AIRFLOW__WEBSERVER__EXPOSE_CONFIG value: "True" - name: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT