Feat/704 705 706 707 terraform improvements

infrastructure/README.md

@@ -8,12 +8,20 @@ This directory contains all infrastructure definitions for PredictIQ using Terra
 infrastructure/
 ├── terraform/
 │   ├── main.tf              # Main configuration
-│   ├── variables.tf         # Variable definitions
+│   ├── variables.tf         # Variable definitions with validation
 │   ├── outputs.tf           # Output definitions
+│   ├── locals.tf            # Common tags and locals
+│   ├── bootstrap.sh         # Bootstrap script for state backend
+│   ├── backend-config.hcl   # Default backend configuration
 │   ├── environments/        # Environment-specific configurations
-│   │   ├── dev.tfvars
-│   │   ├── staging.tfvars
-│   │   └── prod.tfvars
+│   │   ├── README.md        # Environment separation documentation
+│   │   ├── dev.tfvars       # Development environment variables
+│   │   ├── staging/
+│   │   │   ├── terraform.tfvars
+│   │   │   └── backend.hcl
+│   │   └── production/
+│   │       ├── terraform.tfvars
+│   │       └── backend.hcl
 │   └── modules/             # Reusable modules
 │       ├── vpc/
 │       ├── rds/
@@ -33,36 +41,74 @@ infrastructure/
 
 ## Quick Start
 
+### Bootstrap Terraform State Backend (First Time Only)
+
+Before initializing Terraform, you must create the S3 bucket and DynamoDB table for remote state management:
+
+```bash
+cd infrastructure/terraform
+
+# Bootstrap for development environment
+./bootstrap.sh us-east-1 dev
+
+# Bootstrap for staging environment
+./bootstrap.sh us-east-1 staging
+
+# Bootstrap for production environment
+./bootstrap.sh us-east-1 prod
+```
+
+The bootstrap script will:
+1. Create an S3 bucket for Terraform state
+2. Enable versioning and encryption on the bucket
+3. Block public access to the bucket
+4. Create a DynamoDB table for state locking
+5. Enable point-in-time recovery on the DynamoDB table
+
 ### Initialize Terraform
 
 ```bash
 cd infrastructure/terraform
-terraform init
+
+# Initialize with backend configuration
+terraform init -backend-config=backend-config.hcl
 ```
 
+**Note:** The `backend-config.hcl` file contains the S3 bucket and DynamoDB table names. Update this file if you used different names during bootstrap.
+
 ### Plan Infrastructure Changes
 
 ```bash
 # For development environment
 terraform plan -var-file="environments/dev.tfvars"
 
 # For staging environment
-terraform plan -var-file="environments/staging.tfvars"
+terraform plan -var-file="environments/staging/terraform.tfvars"
 
 # For production environment
-terraform plan -var-file="environments/prod.tfvars"
+terraform plan -var-file="environments/production/terraform.tfvars"
 ```
 
 ### Apply Infrastructure Changes
 
 ```bash
 # Apply changes (requires approval)
-terraform apply -var-file="environments/prod.tfvars"
+terraform apply -var-file="environments/production/terraform.tfvars"
 
 # Auto-approve (use with caution)
-terraform apply -auto-approve -var-file="environments/prod.tfvars"
+terraform apply -auto-approve -var-file="environments/production/terraform.tfvars"
 ```
 
+## Environment Separation
+
+PredictIQ uses separate Terraform state files and backends for each environment:
+
+- **Development**: Local state (for testing only)
+- **Staging**: Remote state in S3 with DynamoDB locking
+- **Production**: Remote state in separate S3 bucket with DynamoDB locking
+
+See `environments/README.md` for detailed environment management instructions.
+
 ## Environments
 
 ### Development (dev)

infrastructure/terraform/backend-config.hcl

@@ -0,0 +1,9 @@
+# Backend configuration for Terraform state management
+# This file is used during terraform init to configure the S3 backend
+# Usage: terraform init -backend-config=backend-config.hcl
+
+bucket         = "predictiq-terraform-state"
+key            = "terraform.tfstate"
+region         = "us-east-1"
+encrypt        = true
+dynamodb_table = "terraform-locks"

infrastructure/terraform/bootstrap.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -e
+
+# Bootstrap script to create S3 bucket and DynamoDB table for Terraform state management
+# Usage: ./bootstrap.sh <aws-region> <environment>
+
+AWS_REGION=${1:-us-east-1}
+ENVIRONMENT=${2:-dev}
+BUCKET_NAME="predictiq-terraform-state-${ENVIRONMENT}"
+LOCK_TABLE="terraform-locks-${ENVIRONMENT}"
+
+echo "Bootstrapping Terraform state backend for environment: $ENVIRONMENT in region: $AWS_REGION"
+
+# Create S3 bucket
+echo "Creating S3 bucket: $BUCKET_NAME"
+aws s3api create-bucket \
+  --bucket "$BUCKET_NAME" \
+  --region "$AWS_REGION" \
+  $([ "$AWS_REGION" != "us-east-1" ] && echo "--create-bucket-configuration LocationConstraint=$AWS_REGION") \
+  2>/dev/null || echo "Bucket already exists or error occurred"
+
+# Enable versioning
+echo "Enabling versioning on S3 bucket"
+aws s3api put-bucket-versioning \
+  --bucket "$BUCKET_NAME" \
+  --versioning-configuration Status=Enabled \
+  --region "$AWS_REGION"
+
+# Enable encryption
+echo "Enabling server-side encryption on S3 bucket"
+aws s3api put-bucket-encryption \
+  --bucket "$BUCKET_NAME" \
+  --server-side-encryption-configuration '{
+    "Rules": [
+      {
+        "ApplyServerSideEncryptionByDefault": {
+          "SSEAlgorithm": "AES256"
+        }
+      }
+    ]
+  }' \
+  --region "$AWS_REGION"
+
+# Block public access
+echo "Blocking public access to S3 bucket"
+aws s3api put-public-access-block \
+  --bucket "$BUCKET_NAME" \
+  --public-access-block-configuration \
+  "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" \
+  --region "$AWS_REGION"
+
+# Create DynamoDB table for state locking
+echo "Creating DynamoDB table: $LOCK_TABLE"
+aws dynamodb create-table \
+  --table-name "$LOCK_TABLE" \
+  --attribute-definitions AttributeName=LockID,AttributeType=S \
+  --key-schema AttributeName=LockID,KeyType=HASH \
+  --billing-mode PAY_PER_REQUEST \
+  --region "$AWS_REGION" \
+  2>/dev/null || echo "Table already exists or error occurred"
+
+# Enable point-in-time recovery
+echo "Enabling point-in-time recovery on DynamoDB table"
+aws dynamodb update-continuous-backups \
+  --table-name "$LOCK_TABLE" \
+  --point-in-time-recovery-specification PointInTimeRecoveryEnabled=true \
+  --region "$AWS_REGION" \
+  2>/dev/null || echo "PITR already enabled or error occurred"
+
+echo "Bootstrap complete!"
+echo "S3 Bucket: $BUCKET_NAME"
+echo "DynamoDB Table: $LOCK_TABLE"
+echo ""
+echo "Next steps:"
+echo "1. Update infrastructure/terraform/backend-config.hcl with the bucket and table names"
+echo "2. Run: terraform init -backend-config=backend-config.hcl"

infrastructure/terraform/environments/README.md

@@ -0,0 +1,167 @@
+# Terraform Environments
+
+This directory contains environment-specific configurations for PredictIQ infrastructure.
+
+## Directory Structure
+
+```
+environments/
+├── dev.tfvars                    # Development environment variables
+├── staging/
+│   ├── terraform.tfvars         # Staging environment variables
+│   └── backend.hcl              # Staging backend configuration
+└── production/
+    ├── terraform.tfvars         # Production environment variables
+    └── backend.hcl              # Production backend configuration
+```
+
+## Environment Separation
+
+Each environment has:
+- **Separate state files**: Stored in different S3 buckets with distinct keys
+- **Separate DynamoDB tables**: For state locking to prevent concurrent modifications
+- **Distinct resource naming**: All resources are prefixed with environment name
+- **Different resource sizing**: Production has higher capacity than staging
+
+## Deployment Instructions
+
+### Development Environment
+
+```bash
+cd infrastructure/terraform
+terraform init
+terraform plan -var-file="environments/dev.tfvars"
+terraform apply -var-file="environments/dev.tfvars"
+```
+
+### Staging Environment
+
+```bash
+cd infrastructure/terraform
+
+# First time: bootstrap the backend
+./bootstrap.sh us-east-1 staging
+
+# Initialize with staging backend
+terraform init -backend-config=environments/staging/backend.hcl
+
+# Plan and apply
+terraform plan -var-file="environments/staging/terraform.tfvars"
+terraform apply -var-file="environments/staging/terraform.tfvars"
+```
+
+### Production Environment
+
+```bash
+cd infrastructure/terraform
+
+# First time: bootstrap the backend
+./bootstrap.sh us-east-1 production
+
+# Initialize with production backend
+terraform init -backend-config=environments/production/backend.hcl
+
+# Plan and apply (requires explicit approval)
+terraform plan -var-file="environments/production/terraform.tfvars"
+terraform apply -var-file="environments/production/terraform.tfvars"
+```
+
+## CI/CD Deployment
+
+### Staging Deployment
+
+Staging deployments are automatic on merge to `main` branch:
+
+```yaml
+- name: Deploy to Staging
+  run: |
+    cd infrastructure/terraform
+    terraform init -backend-config=environments/staging/backend.hcl
+    terraform plan -var-file="environments/staging/terraform.tfvars"
+    terraform apply -auto-approve -var-file="environments/staging/terraform.tfvars"
+```
+
+### Production Deployment
+
+Production deployments require explicit approval:
+
+```yaml
+- name: Plan Production Changes
+  run: |
+    cd infrastructure/terraform
+    terraform init -backend-config=environments/production/backend.hcl
+    terraform plan -var-file="environments/production/terraform.tfvars" -out=tfplan
+
+- name: Approve and Apply Production
+  if: github.event_name == 'workflow_dispatch'
+  run: |
+    cd infrastructure/terraform
+    terraform apply tfplan
+```
+
+## State File Locations
+
+| Environment | S3 Bucket | DynamoDB Table | State Key |
+|-------------|-----------|----------------|-----------|
+| Development | Local | N/A | N/A |
+| Staging | `predictiq-terraform-state-staging` | `terraform-locks-staging` | `staging/terraform.tfstate` |
+| Production | `predictiq-terraform-state-production` | `terraform-locks-production` | `production/terraform.tfstate` |
+
+## Important Notes
+
+### Preventing Accidental Production Changes
+
+1. **State Locking**: DynamoDB tables prevent concurrent modifications
+2. **Separate Backends**: Production state is isolated from staging
+3. **CI/CD Approval**: Production changes require manual approval
+4. **Resource Naming**: All resources include environment prefix (e.g., `predictiq-prod-vpc`)
+
+### Switching Environments
+
+When switching between environments, always reinitialize Terraform:
+
+```bash
+# Switch from staging to production
+terraform init -backend-config=environments/production/backend.hcl -reconfigure
+
+# Switch from production to staging
+terraform init -backend-config=environments/staging/backend.hcl -reconfigure
+```
+
+### Disaster Recovery
+
+If state is corrupted:
+
+1. **Staging**: Can be recreated from scratch
+2. **Production**: Contact infrastructure team before any recovery action
+
+```bash
+# Force unlock if state is locked
+terraform force-unlock <LOCK_ID>
+
+# Refresh state from AWS
+terraform refresh -var-file="environments/production/terraform.tfvars"
+```
+
+## Monitoring Environment Health
+
+```bash
+# Check staging resources
+aws ec2 describe-instances --filters "Name=tag:Environment,Values=staging"
+
+# Check production resources
+aws ec2 describe-instances --filters "Name=tag:Environment,Values=prod"
+
+# View state file versions
+aws s3api list-object-versions --bucket predictiq-terraform-state-production
+```
+
+## Best Practices
+
+1. **Always test in staging first** before applying to production
+2. **Review terraform plan output** carefully before applying
+3. **Use terraform workspace** for additional isolation if needed
+4. **Keep backend configurations** in version control (no secrets)
+5. **Enable MFA** for production deployments
+6. **Document all manual changes** made outside Terraform
+7. **Regularly backup state files** using S3 versioning

infrastructure/terraform/environments/production/backend.hcl

@@ -0,0 +1,8 @@
+# Backend configuration for production environment
+# Usage: terraform init -backend-config=environments/production/backend.hcl
+
+bucket         = "predictiq-terraform-state-production"
+key            = "production/terraform.tfstate"
+region         = "us-east-1"
+encrypt        = true
+dynamodb_table = "terraform-locks-production"

infrastructure/terraform/environments/staging/backend.hcl

@@ -0,0 +1,8 @@
+# Backend configuration for staging environment
+# Usage: terraform init -backend-config=environments/staging/backend.hcl
+
+bucket         = "predictiq-terraform-state-staging"
+key            = "staging/terraform.tfstate"
+region         = "us-east-1"
+encrypt        = true
+dynamodb_table = "terraform-locks-staging"

infrastructure/terraform/locals.tf

@@ -0,0 +1,9 @@
+# Common locals for consistent tagging across all modules
+locals {
+  common_tags = {
+    Project   = "predictiq"
+    Environment = var.environment
+    Owner     = "infrastructure-team"
+    ManagedBy = "terraform"
+  }
+}