diff --git a/modules/aws/aws-infra/.terraform-docs.yml b/modules/aws/aws-infra/.terraform-docs.yml new file mode 100644 index 0000000..f2ae68a --- /dev/null +++ b/modules/aws/aws-infra/.terraform-docs.yml @@ -0,0 +1,27 @@ +formatter: "markdown table" + +output: + file: "README.md" + mode: inject + template: |- + + {{ .Content }} + + +sections: + show: + - inputs + - outputs + +settings: + anchor: true + color: true + default: true + description: true + escape: true + hide-empty: false + html: true + indent: 2 + required: true + sensitive: true + type: true diff --git a/modules/aws/aws-infra/README.md b/modules/aws/aws-infra/README.md new file mode 100644 index 0000000..c11158f --- /dev/null +++ b/modules/aws/aws-infra/README.md @@ -0,0 +1,393 @@ +# AWS Infrastructure Module for Databricks + +A comprehensive, production-ready AWS infrastructure module that provides all necessary resources for Databricks workloads using official AWS Terraform modules and best practices. + +## Overview + +This module creates a complete AWS infrastructure foundation optimized for Databricks, featuring: + +- **🔧 Simplified Configuration**: Uses official `terraform-aws-modules/vpc` for networking +- **🔒 Secure Storage**: S3 buckets with configurable encryption (SSE-S3 default, or SSE-KMS with your own key) +- **👤 IAM Integration**: Cross-account and Unity Catalog roles with Databricks-generated policies; cross-account policy type is configurable (`managed`, `restricted`, or `customer-managed`) +- **🔗 VPC Endpoints**: Private access to AWS services (S3, STS, Kinesis) +- **🛡️ Network Firewall**: Configurable FQDN and network-based filtering (optional) +- **🌐 Hub-Spoke Architecture**: Transit Gateway with centralized internet egress (optional) +- **🔐 Private Link**: Databricks Private Link endpoints (optional) + +## Architecture + +### Basic Architecture + +``` +┌─────────────────────────────────────────────┐ +│ VPC (10.0.0.0/16) │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Private Sub │ │ Private Sub │ │ +│ │ (AZ-a) │ │ (AZ-b) │ │ +│ │ Databricks │ │ Databricks │ │ +│ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ +│ └─────────┬────────┘ │ +│ │ │ +│ ┌─────────▼─────────┐ │ +│ │ NAT Gateway │ │ +│ │ (Public Subnet) │ │ +│ └─────────┬─────────┘ │ +│ │ │ +│ ┌─────────▼─────────┐ │ +│ │ Internet Gateway │ │ +│ └───────────────────┘ │ +└─────────────────────────────────────────────┘ + │ + ▼ + Internet +``` + +### Hub-Spoke Architecture with Firewall + +``` +┌──────────────────────────────────────────────────┐ +│ Spoke VPC (Databricks - 10.0.0.0/16) │ +│ Private Subnets │ +└────────────────┬─────────────────────────────────┘ + │ Transit Gateway + ▼ +┌──────────────────────────────────────────────────┐ +│ Hub VPC (10.1.0.0/16) │ +│ │ +│ Private Subnet → Network Firewall → NAT → IGW │ +│ (TGW attach) (Inspection) │ +└──────────────────────────────────────────────────┘ + │ + ▼ + Internet +``` + +## Module Components + +### Core Components (Always Created) +- **networking.tf** - VPC, subnets, security groups, NAT gateway (via AWS VPC module). When hub-spoke is enabled, NAT is automatically disabled in the spoke — the hub VPC handles all egress. +- **workspacestorage.tf** - Root S3 bucket for Databricks workspace +- **ucstorage.tf** - Unity Catalog S3 buckets (metastore & data) +- **iam.tf** - IAM roles (cross-account, Unity Catalog, optional instance profiles) +- **vpc-endpoints.tf** - VPC endpoints (S3, STS, Kinesis) via AWS module + +### Conditional Components +- **private-link.tf** - Databricks Private Link (when `enable_private_link = true`) + +### Submodules +- **modules/hub-networking/** - Transit Gateway, Hub VPC, and Network Firewall (when `hub_spoke_architecture = true`) + +## Usage Examples + +### Minimal Configuration + +```hcl +module "databricks_infra" { + source = "./modules/aws/aws-infra" + + prefix = "my-databricks" + region = "us-west-2" + + networking = { + vpc_cidr = "10.0.0.0/16" + availability_zones = ["us-west-2a", "us-west-2b"] + enable_nat_gateway = true + } + + databricks_account_id = "414351767826" # Databricks AWS account + + tags = { + Environment = "production" + } +} +``` + +### With Hub-Spoke and Network Firewall + +```hcl +module "databricks_infra" { + source = "./modules/aws/aws-infra" + + prefix = "my-databricks" + region = "us-west-2" + + networking = { + vpc_cidr = "10.0.0.0/16" + availability_zones = ["us-west-2a", "us-west-2b"] + enable_nat_gateway = true + } + + databricks_account_id = "414351767826" + + # Hub-Spoke Architecture with Firewall + advanced_networking = { + hub_spoke_architecture = true + enable_transit_gateway = true + hub_vpc_cidr = "10.1.0.0/16" + } + + # Network Firewall Configuration + security = { + enable_network_firewall = true + + # Allow specific domains + # Note: AWS Network Firewall uses leading-dot format for subdomain matching + allowed_fqdns = [ + ".cloud.databricks.com", + ".s3.us-west-2.amazonaws.com", + ".pypi.org", + "files.pythonhosted.org", + "github.com" + ] + + # Allow specific network rules + allowed_network_rules = [ + { + protocol = "TCP" + source_ip = "$HOME_NET" + destination_ip = "ANY" + destination_port = "443" + }, + { + protocol = "UDP" + source_ip = "$HOME_NET" + destination_ip = "ANY" + destination_port = "53" + } + ] + } + + tags = { + Environment = "production" + } +} +``` + +### With Private Link + +```hcl +module "databricks_infra" { + source = "./modules/aws/aws-infra" + + prefix = "my-databricks" + region = "us-west-2" + + networking = { + vpc_cidr = "10.0.0.0/16" + availability_zones = ["us-west-2a", "us-west-2b"] + enable_nat_gateway = false # Not needed with Private Link + } + + databricks_account_id = "414351767826" + + # Private Link Configuration + security = { + enable_private_link = true + backend_service_name = "com.amazonaws.vpce.us-west-2.vpce-svc-0158114c0c730c3bb" + relay_service_name = "com.amazonaws.vpce.us-west-2.vpce-svc-0dc0e98e4e8a7d1f9" + } + + tags = { + Environment = "production" + } +} +``` + +### With Unity Catalog + +```hcl +module "databricks_infra" { + source = "./modules/aws/aws-infra" + + prefix = "my-databricks" + region = "us-west-2" + + networking = { + vpc_cidr = "10.0.0.0/16" + availability_zones = ["us-west-2a", "us-west-2b"] + enable_nat_gateway = true + } + + databricks_account_id = "414351767826" + + # Unity Catalog Configuration + create_metastore_bucket = true + unity_catalog_account_id = "414351767826" + external_id = "12345678-1234-1234-1234-123456789abc" + + tags = { + Environment = "production" + } +} +``` + +> **Note**: The inputs/outputs table below is generated by [terraform-docs](https://terraform-docs.io/) via `.terraform-docs.yml`. Run `terraform-docs .` from this directory to regenerate it after variable changes. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [advanced\_networking](#input\_advanced\_networking) | Advanced networking features |
object({
# Transit Gateway
enable_transit_gateway = optional(bool, false)
hub_spoke_architecture = optional(bool, false)

# Hub VPC configuration (when hub-spoke enabled)
hub_vpc_cidr = optional(string, "10.1.0.0/16")

# Additional VPC attachments
additional_vpc_attachments = optional(list(object({
vpc_id = string
vpc_cidr = string
route_cidr = string
subnet_ids = list(string)
})), [])

# Routing configuration
propagate_default_routes = optional(bool, false)
enable_dns_support = optional(bool, true)
})
| `{}` | no | +| [create\_instance\_profiles](#input\_create\_instance\_profiles) | Create IAM instance profiles for Databricks clusters | `bool` | `false` | no | +| [create\_metastore\_bucket](#input\_create\_metastore\_bucket) | Create Unity Catalog metastore bucket | `bool` | `false` | no | +| [cross\_account\_policy\_type](#input\_cross\_account\_policy\_type) | Databricks cross-account IAM policy type. Options: 'managed' (default AWS-managed policy), 'restricted' (least-privilege), 'customer-managed' (customer-managed VPC) | `string` | `"managed"` | no | +| [databricks\_account\_id](#input\_databricks\_account\_id) | Databricks Account ID (UUID). Found at accounts.cloud.databricks.com → top-right menu. Used to scope the cross-account IAM role trust policy to your Databricks account only. | `string` | n/a | yes | +| [databricks\_config](#input\_databricks\_config) | Databricks-specific configuration for policy generation |
object({
account_id = optional(string, null)
# This helps generate proper Databricks policies but doesn't create Databricks resources
})
| `{}` | no | +| [external\_id](#input\_external\_id) | External ID for Unity Catalog IAM role trust relationship. When null, a basic trust policy (no ExternalId condition) is used. Set and re-apply once available. | `string` | `null` | no | +| [networking](#input\_networking) | VPC and networking configuration |
object({
vpc_cidr = string
availability_zones = optional(list(string), [])
enable_nat_gateway = optional(bool, true)
private_subnet_cidrs = optional(list(string), [])
public_subnet_cidrs = optional(list(string), [])
})
| n/a | yes | +| [prefix](#input\_prefix) | Prefix for all AWS resources | `string` | n/a | yes | +| [region](#input\_region) | AWS region for resource deployment | `string` | n/a | yes | +| [roles\_to\_assume](#input\_roles\_to\_assume) | Additional IAM role ARNs that the cross-account role should be able to assume | `list(string)` | `[]` | no | +| [security](#input\_security) | Advanced security configuration |
object({
# Firewall configuration
enable_network_firewall = optional(bool, false)
allowed_fqdns = optional(list(string), [])
allowed_network_rules = optional(list(object({
protocol = string
source_ip = string
destination_ip = string
destination_port = string
})), [])

# Private Link configuration
enable_private_link = optional(bool, false)
backend_service_name = optional(string, null)
relay_service_name = optional(string, null)
})
| `{}` | no | +| [storage\_encryption](#input\_storage\_encryption) | S3 bucket encryption configuration. Use 'SSE-S3' for AWS-managed keys or 'SSE-KMS' for KMS-managed keys. |
object({
type = optional(string, "SSE-S3")
kms_key_id = optional(string, null)
})
| `{}` | no | +| [tags](#input\_tags) | Common tags for all resources | `map(string)` | `{}` | no | +| [unity\_catalog\_account\_id](#input\_unity\_catalog\_account\_id) | Unity Catalog AWS account ID (Databricks account for Unity Catalog) | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cross\_account\_role\_arn](#output\_cross\_account\_role\_arn) | ARN of the cross-account IAM role for Databricks | +| [cross\_account\_role\_name](#output\_cross\_account\_role\_name) | Name of the cross-account IAM role | +| [data\_bucket\_name](#output\_data\_bucket\_name) | Name of the Unity Catalog data bucket | +| [metastore\_bucket\_name](#output\_metastore\_bucket\_name) | Name of the Unity Catalog metastore bucket (if created) | +| [root\_bucket\_name](#output\_root\_bucket\_name) | Name of the root storage bucket | +| [unity\_catalog\_role\_arn](#output\_unity\_catalog\_role\_arn) | ARN of the Unity Catalog IAM role | +| [unity\_catalog\_role\_name](#output\_unity\_catalog\_role\_name) | Name of the Unity Catalog IAM role | +| [vpc\_id](#output\_vpc\_id) | ID of the Spoke VPC | + + +## Network Firewall Rules + +### FQDN Rules +The firewall uses domain-based filtering to allow/deny traffic based on FQDNs. Pass your allowed domains via `security.allowed_fqdns`. + +> **Important**: AWS Network Firewall `rules_source_list` uses a leading-dot format (`.domain.com`) to match a domain and all its subdomains. The wildcard format (`*.domain.com`) is **not** supported and will cause an `InvalidRequestException`. + +```hcl +allowed_fqdns = [ + ".cloud.databricks.com", + ".s3.us-west-2.amazonaws.com", + ".pypi.org", + "files.pythonhosted.org", + "repo1.maven.org", + "github.com" +] +``` + +### Network Rules +For IP/Protocol/Port-based rules, use `security.allowed_network_rules`: + +```hcl +allowed_network_rules = [ + { + protocol = "TCP" + source_ip = "$HOME_NET" + destination_ip = "ANY" + destination_port = "443" + }, + { + protocol = "UDP" + source_ip = "$HOME_NET" + destination_ip = "ANY" + destination_port = "53" + } +] +``` + +### Default Deny +The firewall includes a default deny rule at the lowest priority. Only explicitly allowed traffic passes through. + +## Traffic Flow + +### Hub-Spoke with Firewall + +1. **Spoke VPC Private Subnet** → Route to Hub VPC via Transit Gateway +2. **Transit Gateway** → Forward to Hub VPC Private Subnet +3. **Hub Private Subnet** → Route to NAT Gateway +4. **NAT Gateway** → Performs SNAT +5. **Hub Public Subnet** → Route to Firewall (if enabled) or IGW +6. **Network Firewall** → Inspect traffic (FQDN, IP, Port rules) +7. **Firewall Subnet** → Route to Internet Gateway +8. **Internet Gateway** → Forward to internet + +## Module Dependencies + +This module uses the following official AWS Terraform modules: + +- **[terraform-aws-modules/vpc/aws](https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws)** (~> 5.0) + - VPC, subnets, NAT Gateway, Internet Gateway, route tables +- **[terraform-aws-modules/vpc/aws//modules/vpc-endpoints](https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws)** (~> 5.0) + - VPC endpoints for S3, STS, Kinesis + +## Provider Requirements + +```hcl +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.57.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.0.0" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +} +``` + +## Best Practices + +### Security +- ✅ Use Private Link for maximum security and reduced data egress costs +- ✅ Enable Network Firewall with allowlist-based FQDN rules +- ✅ Use Unity Catalog IAM roles with least privilege +- ✅ Enable VPC endpoints for S3, STS, and Kinesis + +### Networking +- ✅ Use hub-spoke architecture for centralized internet egress and inspection +- ✅ Deploy NAT Gateway for private subnet internet access +- ✅ Use multiple availability zones for high availability +- ✅ Implement proper subnet sizing for growth + +### Cost Optimization +- ✅ Use single NAT Gateway (default) instead of per-AZ for dev/test +- ✅ Consider Private Link to reduce data egress costs +- ✅ Use VPC endpoints to avoid internet gateway data transfer charges + +## Troubleshooting + +### Common Issues + +**Issue**: Terraform validation fails with "Reference to undeclared resource" +- **Solution**: Run `terraform init -upgrade` to download required modules + +**Issue**: Network Firewall blocks Databricks traffic +- **Solution**: Ensure `allowed_fqdns` includes `*.cloud.databricks.com` and required AWS services + +**Issue**: Unity Catalog role trust relationship fails +- **Solution**: Verify `external_id` matches your Databricks Unity Catalog configuration + +**Issue**: Private Link endpoints not accessible +- **Solution**: Check security group rules allow traffic from Databricks subnets on ports 443, 5432, 8443-8451 + +## Support + +For issues, questions, or contributions: +- Open an issue in the repository +- Refer to [Databricks AWS documentation](https://docs.databricks.com/administration-guide/cloud-configurations/aws/index.html) +- Check [AWS VPC module documentation](https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws) + +## License + +This module is provided as-is for use with Databricks on AWS. diff --git a/modules/aws/aws-infra/iam.tf b/modules/aws/aws-infra/iam.tf new file mode 100644 index 0000000..60d2429 --- /dev/null +++ b/modules/aws/aws-infra/iam.tf @@ -0,0 +1,111 @@ +# IAM Component +# Creates cross-account roles, Unity Catalog roles, and associated policies + +# Databricks-generated Cross-Account Assume Role Policy +data "databricks_aws_assume_role_policy" "cross_account" { + external_id = var.databricks_account_id +} + +# Cross-Account Role for Databricks (Always created) +resource "aws_iam_role" "cross_account" { + name = local.iam_config.cross_account_role_name + assume_role_policy = data.databricks_aws_assume_role_policy.cross_account.json + + tags = merge(local.common_tags, { + Name = local.iam_config.cross_account_role_name + Purpose = "Databricks Cross-Account Access" + Type = "CrossAccount" + }) +} + +# Databricks-generated Cross-Account Policy +# policy_type options: "managed" (default), "restricted", "customer-managed" +data "databricks_aws_crossaccount_policy" "cross_account" { + policy_type = var.cross_account_policy_type + pass_roles = length(var.roles_to_assume) > 0 ? var.roles_to_assume : null +} + +# Attach policy to cross-account role +resource "aws_iam_role_policy" "cross_account_inline" { + name = "databricks-cross-account-policy" + role = aws_iam_role.cross_account.id + policy = data.databricks_aws_crossaccount_policy.cross_account.json +} + +# Unity Catalog IAM role is always created. +# When external_id is not yet known, a basic trust policy (no ExternalId condition) is used. +# Once you have the external_id from the Databricks Account Console, set it and re-apply — +# Terraform will update the trust policy in-place without recreating the role. + +data "databricks_aws_unity_catalog_assume_role_policy" "unity_catalog" { + count = var.external_id != null ? 1 : 0 + aws_account_id = local.account_id + role_name = local.iam_config.unity_catalog_role_name + external_id = var.external_id +} + +# Fallback trust policy used when external_id is not yet available +data "aws_iam_policy_document" "unity_catalog_assume_role_basic" { + count = var.external_id == null ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = ["arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"] + } + } +} + +resource "aws_iam_role" "unity_catalog" { + name = local.iam_config.unity_catalog_role_name + assume_role_policy = var.external_id != null ? ( + data.databricks_aws_unity_catalog_assume_role_policy.unity_catalog[0].json + ) : ( + data.aws_iam_policy_document.unity_catalog_assume_role_basic[0].json + ) + + tags = merge(local.common_tags, { + Name = local.iam_config.unity_catalog_role_name + Purpose = "Unity Catalog Metastore Access" + Type = "UnityCatalog" + }) +} + +data "databricks_aws_unity_catalog_policy" "unity_catalog" { + aws_account_id = local.account_id + role_name = local.iam_config.unity_catalog_role_name + bucket_name = var.create_metastore_bucket ? aws_s3_bucket.metastore[0].bucket : "" +} + +resource "aws_iam_role_policy" "unity_catalog_inline" { + name = "unity-catalog-metastore-policy" + role = aws_iam_role.unity_catalog.id + policy = data.databricks_aws_unity_catalog_policy.unity_catalog.json +} + +# Instance Profiles (optional) +resource "aws_iam_instance_profile" "databricks" { + count = var.create_instance_profiles ? 1 : 0 + + name = "${var.prefix}-databricks-instance-profile" + role = aws_iam_role.cross_account.name + + tags = merge(local.common_tags, { + Name = "${var.prefix}-databricks-instance-profile" + Purpose = "Databricks Compute Instance Profile" + }) +} + +# Wait for IAM role propagation before dependent resources use the roles +resource "time_sleep" "iam_propagation_wait" { + create_duration = "20s" + + depends_on = [ + aws_iam_role.cross_account, + aws_iam_role_policy.cross_account_inline, + aws_iam_role.unity_catalog, + aws_iam_role_policy.unity_catalog_inline, + ] +} diff --git a/modules/aws/aws-infra/locals.tf b/modules/aws/aws-infra/locals.tf new file mode 100644 index 0000000..bf23125 --- /dev/null +++ b/modules/aws/aws-infra/locals.tf @@ -0,0 +1,73 @@ +# Data sources +data "aws_availability_zones" "available" { + state = "available" +} + +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} + +# Captures the timestamp once at resource creation time and remains static thereafter, +# preventing unnecessary plan diffs on every apply. +resource "time_static" "created" {} + +locals { + # Common tags applied to all resources + common_tags = merge(var.tags, { + "ManagedBy" = "terraform" + "Module" = "aws-infra" + "Prefix" = var.prefix + "Region" = var.region + "CreatedDate" = formatdate("YYYY-MM-DD", time_static.created.rfc3339) + }) + + # Availability Zones + availability_zones = length(var.networking.availability_zones) > 0 ? var.networking.availability_zones : slice(data.aws_availability_zones.available.names, 0, min(length(data.aws_availability_zones.available.names), 3)) + + # Subnet CIDR calculations + private_subnet_cidrs = length(var.networking.private_subnet_cidrs) > 0 ? var.networking.private_subnet_cidrs : [ + for i in range(length(local.availability_zones)) : cidrsubnet(var.networking.vpc_cidr, 8, i + 1) + ] + + public_subnet_cidrs = length(var.networking.public_subnet_cidrs) > 0 ? var.networking.public_subnet_cidrs : [ + for i in range(length(local.availability_zones)) : cidrsubnet(var.networking.vpc_cidr, 8, i + 101) + ] + + # Storage configuration - hardcoded bucket names + root_bucket_name = "${var.prefix}-rootbucket" + metastore_bucket_name = "${var.prefix}-metastore" + data_bucket_name = "${var.prefix}-data" + + + # IAM configuration + iam_config = { + cross_account_role_name = "${var.prefix}-cross-account-role" + unity_catalog_role_name = "${var.prefix}-unity-catalog-role" + } + + # Enable firewall if explicitly enabled OR if hub-spoke architecture is enabled + enable_firewall = var.security.enable_network_firewall || var.advanced_networking.hub_spoke_architecture + + # When hub-spoke is enabled the spoke VPC routes egress through the hub, so a + # local NAT gateway is not needed. Callers can still override by setting + # networking.enable_nat_gateway = true explicitly. + enable_nat_gateway = var.advanced_networking.hub_spoke_architecture ? false : var.networking.enable_nat_gateway + + # Advanced networking configuration + transit_gateway_config = var.advanced_networking.enable_transit_gateway ? { + name = "${var.prefix}-transit-gateway" + hub_vpc_cidr = var.advanced_networking.hub_vpc_cidr + spoke_vpc_cidr = var.networking.vpc_cidr + + # Hub VPC subnets (single subnet for each type) + hub_public_subnet_cidr = cidrsubnet(var.advanced_networking.hub_vpc_cidr, 8, 1) + hub_private_subnet_cidr = cidrsubnet(var.advanced_networking.hub_vpc_cidr, 8, 10) + hub_firewall_subnet_cidr = cidrsubnet(var.advanced_networking.hub_vpc_cidr, 8, 20) + } : null + + # Current account ID + account_id = data.aws_caller_identity.current.account_id + + # Current region name + current_region = data.aws_region.current.id +} diff --git a/modules/aws/aws-infra/main.tf b/modules/aws/aws-infra/main.tf new file mode 100644 index 0000000..d63346a --- /dev/null +++ b/modules/aws/aws-infra/main.tf @@ -0,0 +1,48 @@ +# AWS Infrastructure Module +# This module provides comprehensive AWS infrastructure for Databricks workloads +# All .tf files in this directory are automatically loaded by Terraform + +# Core Components (Always Created): +# - networking.tf - VPC, Subnets, Security Groups, NAT Gateway +# - workspacestorage.tf - Root S3 Bucket for Databricks workspace +# - ucstorage.tf - Unity Catalog S3 Buckets (metastore & data) +# - iam.tf - IAM Roles (cross-account, Unity Catalog, instance profiles) +# - vpc-endpoints.tf - VPC Endpoints (S3, STS, Kinesis) + +# Conditional Components (Created based on variables): +# - private-link.tf - Databricks Private Link (when enable_private_link = true) + +# Submodules: +# - modules/hub-networking - Transit Gateway, Hub VPC, and Network Firewall (when hub_spoke_architecture = true) + +# Configuration: +# - variables.tf - Input variables +# - locals.tf - Local values and computed configurations +# - outputs.tf - Module outputs +# - versions.tf - Provider version requirements + +# Hub Networking Module (Transit Gateway + Firewall) +module "hub_networking" { + count = var.advanced_networking.hub_spoke_architecture ? 1 : 0 + source = "./modules/hub-networking" + + prefix = var.prefix + region = var.region + + common_tags = local.common_tags + + # Spoke VPC configuration + spoke_vpc_id = module.vpc.vpc_id + spoke_vpc_cidr = var.networking.vpc_cidr + spoke_private_subnet_ids = module.vpc.private_subnets + spoke_route_table_ids = module.vpc.private_route_table_ids + + # Hub VPC configuration + hub_vpc_cidr = var.advanced_networking.hub_vpc_cidr + availability_zones = local.availability_zones + + # Network Firewall configuration + enable_firewall = local.enable_firewall + allowed_fqdns = var.security.allowed_fqdns + allowed_network_rules = var.security.allowed_network_rules +} diff --git a/modules/aws/aws-infra/modules/hub-networking/firewall.tf b/modules/aws/aws-infra/modules/hub-networking/firewall.tf new file mode 100644 index 0000000..f53c7ab --- /dev/null +++ b/modules/aws/aws-infra/modules/hub-networking/firewall.tf @@ -0,0 +1,172 @@ +# Network Firewall Component +# Creates AWS Network Firewall in the Hub VPC with configurable rule groups for advanced security +# Note: Firewall subnets are created in transit-gateway.tf as part of the Hub VPC + +# Rule Group - Allow FQDNs (Domain-based filtering) +resource "aws_networkfirewall_rule_group" "allow_fqdns" { + count = length(var.allowed_fqdns) > 0 ? 1 : 0 + capacity = 100 + name = "${var.prefix}-allow-fqdns-rg" + type = "STATEFUL" + + rule_group { + rule_variables { + ip_sets { + key = "HOME_NET" + ip_set { + definition = [var.spoke_vpc_cidr] + } + } + } + + rules_source { + # Domain-based rules + rules_source_list { + generated_rules_type = "ALLOWLIST" + target_types = ["TLS_SNI", "HTTP_HOST"] + targets = var.allowed_fqdns + } + } + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-allow-fqdns-rg" + }) +} + +# Rule Group - Allow Network Rules (IP, Protocol, Port based filtering) +resource "aws_networkfirewall_rule_group" "allow_network" { + count = length(var.allowed_network_rules) > 0 ? 1 : 0 + capacity = 100 + name = "${var.prefix}-allow-network-rg" + type = "STATEFUL" + + rule_group { + rule_variables { + ip_sets { + key = "HOME_NET" + ip_set { + definition = [var.spoke_vpc_cidr] + } + } + } + + rules_source { + # Network-level rules from variable + dynamic "stateful_rule" { + for_each = var.allowed_network_rules + content { + action = "PASS" + header { + direction = "FORWARD" + protocol = upper(stateful_rule.value.protocol) + source = stateful_rule.value.source_ip + source_port = "ANY" + destination = stateful_rule.value.destination_ip + destination_port = stateful_rule.value.destination_port + } + rule_option { + keyword = "sid" + settings = [tostring(stateful_rule.key + 1)] + } + } + } + } + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-allow-network-rg" + }) +} + +# Rule Group - Deny All Other Traffic (Default Deny) +resource "aws_networkfirewall_rule_group" "deny_all" { + capacity = 10 + name = "${var.prefix}-deny-all-rg" + type = "STATEFUL" + + rule_group { + rule_variables { + ip_sets { + key = "HOME_NET" + ip_set { + definition = [var.spoke_vpc_cidr] + } + } + } + + rules_source { + stateful_rule { + action = "DROP" + header { + direction = "FORWARD" + protocol = "IP" + source = "$HOME_NET" + source_port = "ANY" + destination = "ANY" + destination_port = "ANY" + } + rule_option { + keyword = "sid" + settings = ["100"] + } + } + } + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-deny-all-rg" + }) +} + +# Firewall Policy +resource "aws_networkfirewall_firewall_policy" "main" { + name = "${var.prefix}-firewall-policy" + + firewall_policy { + # Reference FQDN rule group if FQDNs are provided + dynamic "stateful_rule_group_reference" { + for_each = length(var.allowed_fqdns) > 0 ? [1] : [] + content { + resource_arn = aws_networkfirewall_rule_group.allow_fqdns[0].arn + } + } + + # Reference Network rule group if network rules are provided + dynamic "stateful_rule_group_reference" { + for_each = length(var.allowed_network_rules) > 0 ? [1] : [] + content { + resource_arn = aws_networkfirewall_rule_group.allow_network[0].arn + } + } + + # Deny all - evaluated last under DEFAULT_ACTION_ORDER (specific allow rules win) + stateful_rule_group_reference { + resource_arn = aws_networkfirewall_rule_group.deny_all.arn + } + + # Default action for stateless rules - forward to stateful engine + stateless_default_actions = ["aws:forward_to_sfe"] + stateless_fragment_default_actions = ["aws:forward_to_sfe"] + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-firewall-policy" + }) +} + +# Network Firewall +resource "aws_networkfirewall_firewall" "main" { + name = "${var.prefix}-network-firewall" + firewall_policy_arn = aws_networkfirewall_firewall_policy.main.arn + vpc_id = aws_vpc.hub.id + + # Deploy firewall endpoint in hub VPC firewall subnet + subnet_mapping { + subnet_id = aws_subnet.hub_firewall.id + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-network-firewall" + }) +} diff --git a/modules/aws/aws-infra/modules/hub-networking/locals.tf b/modules/aws/aws-infra/modules/hub-networking/locals.tf new file mode 100644 index 0000000..781b61a --- /dev/null +++ b/modules/aws/aws-infra/modules/hub-networking/locals.tf @@ -0,0 +1,14 @@ +# Hub Networking Module Locals + +locals { + # Transit Gateway configuration + transit_gateway_name = "${var.prefix}-transit-gateway" + + # Hub VPC subnet CIDRs + hub_public_subnet_cidr = cidrsubnet(var.hub_vpc_cidr, 8, 1) + hub_private_subnet_cidr = cidrsubnet(var.hub_vpc_cidr, 8, 10) + hub_firewall_subnet_cidr = cidrsubnet(var.hub_vpc_cidr, 8, 20) + + # Current region (for firewall rules) + current_region = var.region +} diff --git a/modules/aws/aws-infra/modules/hub-networking/outputs.tf b/modules/aws/aws-infra/modules/hub-networking/outputs.tf new file mode 100644 index 0000000..07ee733 --- /dev/null +++ b/modules/aws/aws-infra/modules/hub-networking/outputs.tf @@ -0,0 +1,6 @@ +# Hub Networking Module Outputs + +output "hub_vpc_id" { + description = "ID of the hub VPC" + value = aws_vpc.hub.id +} diff --git a/modules/aws/aws-infra/modules/hub-networking/transit-gateway.tf b/modules/aws/aws-infra/modules/hub-networking/transit-gateway.tf new file mode 100644 index 0000000..5dfc0e9 --- /dev/null +++ b/modules/aws/aws-infra/modules/hub-networking/transit-gateway.tf @@ -0,0 +1,251 @@ +# Transit Gateway Component +# Creates Transit Gateway with hub-spoke architecture for enterprise networking +# Transit Gateway +resource "aws_ec2_transit_gateway" "main" { + description = "Transit Gateway for ${var.prefix}" + default_route_table_association = "disable" + default_route_table_propagation = "disable" + dns_support = "enable" + tags = merge(var.common_tags, { + Name = local.transit_gateway_name + }) +} +# Hub VPC (when hub-spoke architecture is enabled) +resource "aws_vpc" "hub" { + cidr_block = var.hub_vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-vpc" + Type = "Hub" + }) +} +# Hub VPC - Internet Gateway +resource "aws_internet_gateway" "hub" { + vpc_id = aws_vpc.hub.id + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-igw" + }) +} +# Hub VPC - Public Subnet (single) +resource "aws_subnet" "hub_public" { + vpc_id = aws_vpc.hub.id + cidr_block = local.hub_public_subnet_cidr + availability_zone = var.availability_zones[0] + map_public_ip_on_launch = true + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-public-subnet" + Type = "HubPublic" + AZ = var.availability_zones[0] + }) +} +# Hub VPC - Private Subnet (single, for Transit Gateway attachment) +resource "aws_subnet" "hub_private" { + vpc_id = aws_vpc.hub.id + cidr_block = local.hub_private_subnet_cidr + availability_zone = var.availability_zones[0] + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-private-subnet" + Type = "HubPrivate" + AZ = var.availability_zones[0] + }) +} +# Hub VPC - Firewall Subnet (single) +resource "aws_subnet" "hub_firewall" { + vpc_id = aws_vpc.hub.id + cidr_block = local.hub_firewall_subnet_cidr + availability_zone = var.availability_zones[0] + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-firewall-subnet" + Type = "HubFirewall" + AZ = var.availability_zones[0] + }) +} +# Hub VPC - NAT Gateway Elastic IP (single) +resource "aws_eip" "hub_nat" { + domain = "vpc" + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-nat-eip" + }) + depends_on = [aws_internet_gateway.hub] +} +# Hub VPC - NAT Gateway (single) +resource "aws_nat_gateway" "hub" { + allocation_id = aws_eip.hub_nat.id + subnet_id = aws_subnet.hub_public.id + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-nat" + AZ = var.availability_zones[0] + }) + depends_on = [aws_internet_gateway.hub] +} +# Hub VPC Route Tables +resource "aws_route_table" "hub_public" { + vpc_id = aws_vpc.hub.id + + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-public-rt" + Type = "HubPublic" + }) +} + +resource "aws_route_table" "hub_private" { + vpc_id = aws_vpc.hub.id + + route { + cidr_block = var.spoke_vpc_cidr + transit_gateway_id = aws_ec2_transit_gateway.main.id + } + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.hub.id + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-private-rt" + Type = "HubPrivate" + }) + depends_on = [aws_ec2_transit_gateway_vpc_attachment.hub] +} + +resource "aws_route_table" "hub_firewall" { + vpc_id = aws_vpc.hub.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.hub.id + } + + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-firewall-rt" + Type = "HubFirewall" + }) +} +# Hub VPC Route Table Associations +resource "aws_route_table_association" "hub_public" { + subnet_id = aws_subnet.hub_public.id + route_table_id = aws_route_table.hub_public.id +} +resource "aws_route_table_association" "hub_private" { + subnet_id = aws_subnet.hub_private.id + route_table_id = aws_route_table.hub_private.id +} +resource "aws_route_table_association" "hub_firewall" { + subnet_id = aws_subnet.hub_firewall.id + route_table_id = aws_route_table.hub_firewall.id +} + +# Route from Hub Public (NAT location) to Firewall for Internet-bound traffic +resource "aws_route" "hub_public_to_firewall" { + count = var.enable_firewall ? 1 : 0 + + route_table_id = aws_route_table.hub_public.id + destination_cidr_block = "0.0.0.0/0" + vpc_endpoint_id = one([for k, v in aws_networkfirewall_firewall.main.firewall_status[0].sync_states : v.attachment[0].endpoint_id]) + + depends_on = [aws_networkfirewall_firewall.main] +} + +# Route from Hub Public to IGW when firewall is NOT enabled +resource "aws_route" "hub_public_to_igw" { + count = var.enable_firewall ? 0 : 1 + + route_table_id = aws_route_table.hub_public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.hub.id +} + +# Transit Gateway VPC Attachment - Spoke (main VPC) +resource "aws_ec2_transit_gateway_vpc_attachment" "spoke" { + subnet_ids = var.spoke_private_subnet_ids + transit_gateway_id = aws_ec2_transit_gateway.main.id + vpc_id = var.spoke_vpc_id + dns_support = "enable" + tags = merge(var.common_tags, { + Name = "${var.prefix}-spoke-tgw-attachment" + Type = "Spoke" + }) +} +# Transit Gateway VPC Attachment - Hub (if hub-spoke is enabled) +resource "aws_ec2_transit_gateway_vpc_attachment" "hub" { + subnet_ids = [aws_subnet.hub_private.id] + transit_gateway_id = aws_ec2_transit_gateway.main.id + vpc_id = aws_vpc.hub.id + dns_support = "enable" + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-tgw-attachment" + Type = "Hub" + }) +} +# Transit Gateway Route Tables (Custom routing - always required since default propagation is disabled) +resource "aws_ec2_transit_gateway_route_table" "spoke" { + transit_gateway_id = aws_ec2_transit_gateway.main.id + tags = merge(var.common_tags, { + Name = "${var.prefix}-spoke-tgw-rt" + }) +} +resource "aws_ec2_transit_gateway_route_table" "hub" { + transit_gateway_id = aws_ec2_transit_gateway.main.id + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-tgw-rt" + }) +} +# Route Table Associations +resource "aws_ec2_transit_gateway_route_table_association" "spoke" { + transit_gateway_attachment_id = aws_ec2_transit_gateway_vpc_attachment.spoke.id + transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.spoke.id +} +resource "aws_ec2_transit_gateway_route_table_association" "hub" { + transit_gateway_attachment_id = aws_ec2_transit_gateway_vpc_attachment.hub.id + transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.hub.id +} +# Transit Gateway Routes +resource "aws_ec2_transit_gateway_route" "spoke_to_hub" { + destination_cidr_block = var.hub_vpc_cidr + transit_gateway_attachment_id = aws_ec2_transit_gateway_vpc_attachment.hub.id + transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.spoke.id +} +resource "aws_ec2_transit_gateway_route" "hub_to_spoke" { + destination_cidr_block = var.spoke_vpc_cidr + transit_gateway_attachment_id = aws_ec2_transit_gateway_vpc_attachment.spoke.id + transit_gateway_route_table_id = aws_ec2_transit_gateway_route_table.hub.id +} +# Update main VPC route tables to route to Transit Gateway +resource "aws_route" "private_to_tgw" { + count = length(var.spoke_route_table_ids) + route_table_id = var.spoke_route_table_ids[count.index] + destination_cidr_block = var.hub_vpc_cidr + transit_gateway_id = aws_ec2_transit_gateway.main.id + depends_on = [aws_ec2_transit_gateway_vpc_attachment.spoke] +} +# Security Group for Hub VPC +resource "aws_security_group" "hub_default" { + name_prefix = "${var.prefix}-hub-" + vpc_id = aws_vpc.hub.id + description = "Default security group for hub VPC" + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } + ingress { + from_port = 0 + to_port = 65535 + protocol = "tcp" + cidr_blocks = [var.spoke_vpc_cidr] + description = "Allow traffic from spoke VPC" + } + ingress { + from_port = 0 + to_port = 65535 + protocol = "udp" + cidr_blocks = [var.spoke_vpc_cidr] + description = "Allow UDP traffic from spoke VPC" + } + tags = merge(var.common_tags, { + Name = "${var.prefix}-hub-default-sg" + }) +} diff --git a/modules/aws/aws-infra/modules/hub-networking/variables.tf b/modules/aws/aws-infra/modules/hub-networking/variables.tf new file mode 100644 index 0000000..8e1081f --- /dev/null +++ b/modules/aws/aws-infra/modules/hub-networking/variables.tf @@ -0,0 +1,72 @@ +# Hub Networking Module Variables +# This module creates Transit Gateway, Hub VPC, and Network Firewall + +variable "prefix" { + description = "Prefix for resource names" + type = string +} + +variable "region" { + description = "AWS region" + type = string +} + +variable "common_tags" { + description = "Common tags to apply to all resources" + type = map(string) + default = {} +} + +variable "spoke_vpc_id" { + description = "ID of the spoke (main) VPC" + type = string +} + +variable "spoke_vpc_cidr" { + description = "CIDR block of the spoke (main) VPC" + type = string +} + +variable "spoke_private_subnet_ids" { + description = "IDs of the spoke VPC private subnets" + type = list(string) +} + +variable "spoke_route_table_ids" { + description = "IDs of the spoke VPC private route tables" + type = list(string) +} + +variable "hub_vpc_cidr" { + description = "CIDR block for the hub VPC" + type = string +} + +variable "availability_zones" { + description = "List of availability zones" + type = list(string) +} + +variable "enable_firewall" { + description = "Enable Network Firewall in the hub VPC" + type = bool + default = true +} + +variable "allowed_fqdns" { + description = "List of FQDNs to allow through the firewall" + type = list(string) + default = [] +} + +variable "allowed_network_rules" { + description = "List of network-level rules (IP, protocol, port)" + type = list(object({ + protocol = string + source_ip = string + destination_ip = string + destination_port = string + })) + default = [] +} + diff --git a/modules/aws/aws-infra/networking.tf b/modules/aws/aws-infra/networking.tf new file mode 100644 index 0000000..f0d3fd6 --- /dev/null +++ b/modules/aws/aws-infra/networking.tf @@ -0,0 +1,116 @@ +# Networking Component +# Creates VPC, subnets, security groups, NAT gateways, and routing using AWS VPC module + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = "${var.prefix}-vpc" + cidr = var.networking.vpc_cidr + + azs = local.availability_zones + private_subnets = local.private_subnet_cidrs + public_subnets = local.enable_nat_gateway ? local.public_subnet_cidrs : [] + + # DNS + enable_dns_hostnames = true + enable_dns_support = true + + # NAT Gateway — disabled automatically when hub-spoke is active (hub handles egress) + enable_nat_gateway = local.enable_nat_gateway + single_nat_gateway = true + + # Tags + tags = local.common_tags + + vpc_tags = { + Name = "${var.prefix}-vpc" + Type = "Main" + } + + private_subnet_tags = { + Type = "Private" + } + + public_subnet_tags = { + Type = "Public" + } + + private_route_table_tags = { + Type = "Private" + } + + public_route_table_tags = { + Type = "Public" + } + + igw_tags = { + Name = "${var.prefix}-igw" + } + + nat_gateway_tags = { + Name = "${var.prefix}-nat-gateway" + } + + nat_eip_tags = { + Name = "${var.prefix}-nat-eip" + } +} + +# Security Group for Databricks +resource "aws_security_group" "default" { + name_prefix = "${var.prefix}-databricks-" + vpc_id = module.vpc.vpc_id + description = "Security group for Databricks workspace" + + # Databricks-specific egress rules for internal communication + dynamic "egress" { + for_each = toset([443, 2443, 6666, 5432, 8443, 8444, 8445, 8446, 8447, 8448, 8449, 8450, 8451]) + content { + description = "Databricks - Workspace SG - REST (443), Secure Cluster Connectivity (2443/6666), Lakebase PostgreSQL (5432), Compute Plane to Control Plane Internal Calls (8443), Unity Catalog Logging and Lineage Data Streaming (8444), Future Extendability (8445-8451)" + from_port = egress.value + to_port = egress.value + protocol = "tcp" + cidr_blocks = [var.networking.vpc_cidr] + } + } + + # Outbound rules to self (required for Databricks clusters) + egress { + from_port = 0 + to_port = 65535 + protocol = "tcp" + self = true + description = "Allow all internal TCP traffic to self" + } + + egress { + from_port = 0 + to_port = 65535 + protocol = "udp" + self = true + description = "Allow all internal UDP traffic to self" + } + + # Inbound rules from self (required for internal cluster communication) + ingress { + from_port = 0 + to_port = 65535 + protocol = "tcp" + self = true + description = "Allow all internal TCP traffic from self" + } + + ingress { + from_port = 0 + to_port = 65535 + protocol = "udp" + self = true + description = "Allow all internal UDP traffic from self" + } + + tags = merge(local.common_tags, { + Name = "${var.prefix}-databricks-sg" + Type = "Databricks" + }) +} diff --git a/modules/aws/aws-infra/outputs.tf b/modules/aws/aws-infra/outputs.tf new file mode 100644 index 0000000..b135ede --- /dev/null +++ b/modules/aws/aws-infra/outputs.tf @@ -0,0 +1,42 @@ +# VPC Output +output "vpc_id" { + description = "ID of the Spoke VPC" + value = module.vpc.vpc_id +} + +# S3 Bucket Names +output "root_bucket_name" { + description = "Name of the root storage bucket" + value = aws_s3_bucket.root.bucket +} + +output "metastore_bucket_name" { + description = "Name of the Unity Catalog metastore bucket (if created)" + value = var.create_metastore_bucket ? aws_s3_bucket.metastore[0].bucket : null +} + +output "data_bucket_name" { + description = "Name of the Unity Catalog data bucket" + value = aws_s3_bucket.data.bucket +} + +# IAM Roles +output "cross_account_role_arn" { + description = "ARN of the cross-account IAM role for Databricks" + value = aws_iam_role.cross_account.arn +} + +output "cross_account_role_name" { + description = "Name of the cross-account IAM role" + value = aws_iam_role.cross_account.name +} + +output "unity_catalog_role_arn" { + description = "ARN of the Unity Catalog IAM role" + value = aws_iam_role.unity_catalog.arn +} + +output "unity_catalog_role_name" { + description = "Name of the Unity Catalog IAM role" + value = aws_iam_role.unity_catalog.name +} diff --git a/modules/aws/aws-infra/private-link.tf b/modules/aws/aws-infra/private-link.tf new file mode 100644 index 0000000..3318943 --- /dev/null +++ b/modules/aws/aws-infra/private-link.tf @@ -0,0 +1,159 @@ +# Private Link Component +# Creates VPC endpoints for Databricks private connectivity + +# Private Link Subnets (dedicated subnets for Databricks VPC endpoints) +resource "aws_subnet" "private_link" { + count = var.security.enable_private_link ? length(local.availability_zones) : 0 + + vpc_id = module.vpc.vpc_id + cidr_block = cidrsubnet(var.networking.vpc_cidr, 8, count.index + 200) + availability_zone = local.availability_zones[count.index] + + tags = merge(local.common_tags, { + Name = "${var.prefix}-private-link-subnet-${count.index + 1}" + Type = "PrivateLink" + AZ = local.availability_zones[count.index] + }) +} + +# Route Table for Private Link Subnets +resource "aws_route_table" "private_link" { + count = var.security.enable_private_link ? 1 : 0 + + vpc_id = module.vpc.vpc_id + + tags = merge(local.common_tags, { + Name = "${var.prefix}-private-link-rt" + Type = "PrivateLink" + }) +} + +# Route Table Association for Private Link Subnets +resource "aws_route_table_association" "private_link" { + count = var.security.enable_private_link ? length(aws_subnet.private_link) : 0 + + subnet_id = aws_subnet.private_link[count.index].id + route_table_id = aws_route_table.private_link[0].id +} + +# Security Group for Private Link Endpoints +resource "aws_security_group" "private_link" { + count = var.security.enable_private_link ? 1 : 0 + + name_prefix = "${var.prefix}-private-link-" + vpc_id = module.vpc.vpc_id + description = "Security group for Databricks Private Link endpoints" + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + security_groups = [aws_security_group.default.id] + description = "HTTPS from Databricks clusters" + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [var.networking.vpc_cidr] + description = "HTTPS from VPC" + } + + # Extended port range for Databricks communication + ingress { + from_port = 6666 + to_port = 6666 + protocol = "tcp" + security_groups = [aws_security_group.default.id] + description = "Databricks internal communication" + } + + ingress { + from_port = 6666 + to_port = 6666 + protocol = "tcp" + cidr_blocks = [var.networking.vpc_cidr] + description = "Databricks internal communication from VPC" + } + + # PostgreSQL port for Lakebase + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.default.id] + description = "Lakebase PostgreSQL from Databricks clusters" + } + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = [var.networking.vpc_cidr] + description = "Lakebase PostgreSQL from VPC" + } + + # Control Plane, Unity Catalog, and Future Extendability ports + ingress { + from_port = 8443 + to_port = 8451 + protocol = "tcp" + security_groups = [aws_security_group.default.id] + description = "Databricks Control Plane (8443), Unity Catalog (8444), Future Extendability (8445-8451) from clusters" + } + + ingress { + from_port = 8443 + to_port = 8451 + protocol = "tcp" + cidr_blocks = [var.networking.vpc_cidr] + description = "Databricks Control Plane (8443), Unity Catalog (8444), Future Extendability (8445-8451) from VPC" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "All outbound traffic" + } + + tags = merge(local.common_tags, { + Name = "${var.prefix}-private-link-sg" + }) +} + +# Databricks Backend Private Link Endpoint +resource "aws_vpc_endpoint" "backend" { + count = var.security.enable_private_link && var.security.backend_service_name != null ? 1 : 0 + + vpc_id = module.vpc.vpc_id + service_name = var.security.backend_service_name + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.private_link[*].id + security_group_ids = [aws_security_group.private_link[0].id] + private_dns_enabled = false + + tags = merge(local.common_tags, { + Name = "${var.prefix}-databricks-backend-endpoint" + Type = "DatabricksPrivateLink" + }) +} + +# Databricks Relay Private Link Endpoint +resource "aws_vpc_endpoint" "relay" { + count = var.security.enable_private_link && var.security.relay_service_name != null ? 1 : 0 + + vpc_id = module.vpc.vpc_id + service_name = var.security.relay_service_name + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.private_link[*].id + security_group_ids = [aws_security_group.private_link[0].id] + private_dns_enabled = false + + tags = merge(local.common_tags, { + Name = "${var.prefix}-databricks-relay-endpoint" + Type = "DatabricksPrivateLink" + }) +} diff --git a/modules/aws/aws-infra/ucstorage.tf b/modules/aws/aws-infra/ucstorage.tf new file mode 100644 index 0000000..76a8236 --- /dev/null +++ b/modules/aws/aws-infra/ucstorage.tf @@ -0,0 +1,75 @@ +# Unity Catalog S3 Buckets Component +# Creates metastore and data S3 buckets with security best practices + +# Metastore Bucket (for Unity Catalog) +resource "aws_s3_bucket" "metastore" { + count = var.create_metastore_bucket ? 1 : 0 + + bucket = local.metastore_bucket_name + + tags = merge(local.common_tags, { + Name = local.metastore_bucket_name + BucketType = "metastore" + Purpose = "Metastore" + }) +} + +resource "aws_s3_bucket" "data" { + bucket = local.data_bucket_name + + tags = merge(local.common_tags, { + Name = local.data_bucket_name + BucketType = "data" + Purpose = "Data" + }) +} + +# S3 Bucket Server-Side Encryption Configuration - Metastore Bucket +resource "aws_s3_bucket_server_side_encryption_configuration" "metastore" { + count = var.create_metastore_bucket ? 1 : 0 + + bucket = aws_s3_bucket.metastore[0].id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = var.storage_encryption.type == "SSE-KMS" ? "aws:kms" : "AES256" + kms_master_key_id = var.storage_encryption.type == "SSE-KMS" ? var.storage_encryption.kms_key_id : null + } + } +} + +# S3 Bucket Server-Side Encryption Configuration - Data Bucket +resource "aws_s3_bucket_server_side_encryption_configuration" "data" { + bucket = aws_s3_bucket.data.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = var.storage_encryption.type == "SSE-KMS" ? "aws:kms" : "AES256" + kms_master_key_id = var.storage_encryption.type == "SSE-KMS" ? var.storage_encryption.kms_key_id : null + } + } +} + +# Note: Versioning is disabled by default (no versioning configuration resources needed) + +# S3 Bucket Public Access Block - Metastore Bucket +resource "aws_s3_bucket_public_access_block" "metastore" { + count = var.create_metastore_bucket ? 1 : 0 + + bucket = aws_s3_bucket.metastore[0].id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# S3 Bucket Public Access Block - Data Bucket +resource "aws_s3_bucket_public_access_block" "data" { + bucket = aws_s3_bucket.data.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} diff --git a/modules/aws/aws-infra/variables.tf b/modules/aws/aws-infra/variables.tf new file mode 100644 index 0000000..e201edf --- /dev/null +++ b/modules/aws/aws-infra/variables.tf @@ -0,0 +1,165 @@ +# Core Configuration Variables +variable "prefix" { + description = "Prefix for all AWS resources" + type = string +} + +variable "region" { + description = "AWS region for resource deployment" + type = string +} + +variable "tags" { + description = "Common tags for all resources" + type = map(string) + default = {} +} + +# Networking Configuration +variable "networking" { + description = "VPC and networking configuration" + type = object({ + vpc_cidr = string + availability_zones = optional(list(string), []) + enable_nat_gateway = optional(bool, true) + private_subnet_cidrs = optional(list(string), []) + public_subnet_cidrs = optional(list(string), []) + }) +} + +# Storage Configuration - Individual Variables +variable "create_metastore_bucket" { + description = "Create Unity Catalog metastore bucket" + type = bool + default = false +} + +variable "storage_encryption" { + description = "S3 bucket encryption configuration. Use 'SSE-S3' for AWS-managed keys or 'SSE-KMS' for KMS-managed keys." + type = object({ + type = optional(string, "SSE-S3") + kms_key_id = optional(string, null) + }) + default = {} + + validation { + condition = contains(["SSE-S3", "SSE-KMS"], var.storage_encryption.type) + error_message = "storage_encryption.type must be either 'SSE-S3' or 'SSE-KMS'." + } + + validation { + condition = var.storage_encryption.type != "SSE-KMS" || var.storage_encryption.kms_key_id != null + error_message = "storage_encryption.kms_key_id must be set when using SSE-KMS encryption." + } +} + +# IAM Configuration - Split into individual variables + +# Instance Profiles (Optional) +variable "create_instance_profiles" { + description = "Create IAM instance profiles for Databricks clusters" + type = bool + default = false +} + +variable "databricks_account_id" { + description = "Databricks Account ID (UUID). Found at accounts.cloud.databricks.com → top-right menu. Used to scope the cross-account IAM role trust policy to your Databricks account only." + type = string +} + +variable "external_id" { + description = "External ID for Unity Catalog IAM role trust relationship. When null, a basic trust policy (no ExternalId condition) is used. Set and re-apply once available." + type = string + default = null +} + +# Unity Catalog Configuration (Always created) +variable "unity_catalog_account_id" { + description = "Unity Catalog AWS account ID (Databricks account for Unity Catalog)" + type = string + default = null +} + +# Additional IAM Permissions +variable "roles_to_assume" { + description = "Additional IAM role ARNs that the cross-account role should be able to assume" + type = list(string) + default = [] +} + +variable "cross_account_policy_type" { + description = "Databricks cross-account IAM policy type. Options: 'managed' (default AWS-managed policy), 'restricted' (least-privilege), 'customer-managed' (customer-managed VPC)" + type = string + default = "managed" + + validation { + condition = contains(["managed", "restricted", "customer-managed"], var.cross_account_policy_type) + error_message = "cross_account_policy_type must be one of: managed, restricted, customer-managed." + } +} + +# Security Configuration +variable "security" { + description = "Advanced security configuration" + type = object({ + # Firewall configuration + enable_network_firewall = optional(bool, false) + allowed_fqdns = optional(list(string), []) + allowed_network_rules = optional(list(object({ + protocol = string + source_ip = string + destination_ip = string + destination_port = string + })), []) + + # Private Link configuration + enable_private_link = optional(bool, false) + backend_service_name = optional(string, null) + relay_service_name = optional(string, null) + }) + + default = {} +} + +# Advanced Networking Configuration +variable "advanced_networking" { + description = "Advanced networking features" + type = object({ + # Transit Gateway + enable_transit_gateway = optional(bool, false) + hub_spoke_architecture = optional(bool, false) + + # Hub VPC configuration (when hub-spoke enabled) + hub_vpc_cidr = optional(string, "10.1.0.0/16") + + # Additional VPC attachments + additional_vpc_attachments = optional(list(object({ + vpc_id = string + vpc_cidr = string + route_cidr = string + subnet_ids = list(string) + })), []) + + # Routing configuration + propagate_default_routes = optional(bool, false) + enable_dns_support = optional(bool, true) + }) + + default = {} + + validation { + condition = !var.advanced_networking.hub_spoke_architecture || var.advanced_networking.enable_transit_gateway + error_message = "Transit Gateway must be enabled when using hub-spoke architecture." + } +} + +# Data Sources Configuration +variable "databricks_config" { + description = "Databricks-specific configuration for policy generation" + type = object({ + account_id = optional(string, null) + # This helps generate proper Databricks policies but doesn't create Databricks resources + }) + + default = {} +} diff --git a/modules/aws/aws-infra/versions.tf b/modules/aws/aws-infra/versions.tf new file mode 100644 index 0000000..f3a0319 --- /dev/null +++ b/modules/aws/aws-infra/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.57.0" + } + databricks = { + source = "databricks/databricks" + version = ">= 1.0.0" + } + time = { + source = "hashicorp/time" + version = ">= 0.9.0" + } + } +} diff --git a/modules/aws/aws-infra/vpc-endpoints.tf b/modules/aws/aws-infra/vpc-endpoints.tf new file mode 100644 index 0000000..dbf885b --- /dev/null +++ b/modules/aws/aws-infra/vpc-endpoints.tf @@ -0,0 +1,48 @@ +# VPC Endpoints Component +# Creates VPC endpoints for secure private access to AWS services using AWS VPC Endpoints module + +module "vpc_endpoints" { + source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" + version = "~> 5.0" + + vpc_id = module.vpc.vpc_id + + endpoints = { + s3 = { + service = "s3" + service_type = "Gateway" + route_table_ids = concat( + module.vpc.private_route_table_ids, + var.networking.enable_nat_gateway ? module.vpc.public_route_table_ids : [] + ) + tags = { + Name = "${var.prefix}-s3-vpc-endpoint" + Type = "Gateway" + } + } + + sts = { + service = "sts" + service_type = "Interface" + private_dns_enabled = true + subnet_ids = module.vpc.private_subnets + tags = { + Name = "${var.prefix}-sts-vpc-endpoint" + Type = "Interface" + } + } + + kinesis-streams = { + service = "kinesis-streams" + service_type = "Interface" + private_dns_enabled = true + subnet_ids = module.vpc.private_subnets + tags = { + Name = "${var.prefix}-kinesis-streams-vpc-endpoint" + Type = "Interface" + } + } + } + + tags = local.common_tags +} diff --git a/modules/aws/aws-infra/workspacestorage.tf b/modules/aws/aws-infra/workspacestorage.tf new file mode 100644 index 0000000..e436008 --- /dev/null +++ b/modules/aws/aws-infra/workspacestorage.tf @@ -0,0 +1,50 @@ +# Workspace S3 Bucket Component +# Creates root S3 bucket for Databricks workspace with security best practices + +# Root Storage Bucket (for Databricks workspace) - Always created +resource "aws_s3_bucket" "root" { + bucket = local.root_bucket_name + + tags = merge(local.common_tags, { + Name = local.root_bucket_name + BucketType = "root" + Purpose = "Root" + }) +} + +# S3 Bucket Server-Side Encryption Configuration - Root Bucket +resource "aws_s3_bucket_server_side_encryption_configuration" "root" { + bucket = aws_s3_bucket.root.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = var.storage_encryption.type == "SSE-KMS" ? "aws:kms" : "AES256" + kms_master_key_id = var.storage_encryption.type == "SSE-KMS" ? var.storage_encryption.kms_key_id : null + } + } +} + +# Note: Versioning is disabled by default (no versioning configuration resources needed) + +# S3 Bucket Public Access Block - Root Bucket +resource "aws_s3_bucket_public_access_block" "root" { + bucket = aws_s3_bucket.root.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# Databricks-generated Root Bucket Policy +data "databricks_aws_bucket_policy" "root" { + bucket = aws_s3_bucket.root.bucket +} + +# Root Storage Bucket Policy (for Databricks workspace) +resource "aws_s3_bucket_policy" "root" { + bucket = aws_s3_bucket.root.id + policy = data.databricks_aws_bucket_policy.root.json + + depends_on = [aws_s3_bucket_public_access_block.root] +}