feat(backend): implement APM, request validation, DB pool optimizatio…

backend/src/app.module.ts

@@ -8,6 +8,8 @@ import { AuditLogInterceptor } from './common/interceptors/audit-log.interceptor
 import { RequestLoggingInterceptor } from './common/interceptors/request-logging.interceptor';
 import { GracefulShutdownInterceptor } from './common/interceptors/graceful-shutdown.interceptor';
 import { TieredThrottlerGuard } from './common/guards/tiered-throttler.guard';
+import { ApmModule } from './modules/apm/apm.module';
+import { ApmInterceptor } from './modules/apm/apm.interceptor';
 import { CommonModule } from './common/common.module';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { LoggerModule } from 'nestjs-pino';
@@ -47,6 +49,7 @@ import { ConnectionPoolModule } from './common/database/connection-pool.module';
 import { CircuitBreakerModule } from './common/circuit-breaker/circuit-breaker.module';
 import { PostmanModule } from './common/postman/postman.module';
 import { CorrelationIdMiddleware } from './common/middleware/correlation-id.middleware';
+import { RequestValidationMiddleware } from './common/middleware/validation.middleware';
 import { PerformanceModule } from './modules/performance/performance.module';
 import { GracefulShutdownService } from './common/services/graceful-shutdown.service';
 
@@ -95,6 +98,9 @@ const envValidationSchema = Joi.object({
   BACKUP_ENCRYPTION_KEY: Joi.string().length(64).optional(), // 32-byte key as hex
   BACKUP_RETENTION_DAYS: Joi.number().integer().min(1).default(30).optional(),
   BACKUP_TMP_DIR: Joi.string().optional(),
+
+  APM_SAMPLING_RATE: Joi.number().min(0).max(1).default(1.0).optional(),
+  APM_ENABLED: Joi.boolean().default(true).optional(),
 });
 
 @Module({
@@ -209,6 +215,7 @@ const envValidationSchema = Joi.object({
     DataExportModule,
     ConnectionPoolModule,
     CircuitBreakerModule,
+    ApmModule,
     PostmanModule,
     PerformanceModule,
     CommonModule,
@@ -254,10 +261,15 @@ const envValidationSchema = Joi.object({
       provide: APP_INTERCEPTOR,
       useClass: GracefulShutdownInterceptor,
     },
+    {
+      provide: APP_INTERCEPTOR,
+      useClass: ApmInterceptor,
+    },
   ],
 })
 export class AppModule {
   configure(consumer: MiddlewareConsumer) {
     consumer.apply(CorrelationIdMiddleware).forRoutes('*');
+    consumer.apply(RequestValidationMiddleware).forRoutes('*');
   }
 }

backend/src/common/database/connection-pool.config.ts

@@ -1,4 +1,4 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { DataSource } from 'typeorm';
 
@@ -8,42 +8,101 @@ export interface PoolMetrics {
   waitingRequests: number;
   totalConnections: number;
   utilizationPercentage: number;
+  maxPoolSize: number;
+  minPoolSize: number;
   timestamp: Date;
 }
 
+export interface PoolHealthStatus {
+  healthy: boolean;
+  utilizationPercentage: number;
+  waitingRequests: number;
+  latencyMs: number;
+  message: string;
+}
+
 @Injectable()
-export class ConnectionPoolService {
+export class ConnectionPoolService implements OnModuleInit, OnModuleDestroy {
   private readonly logger = new Logger(ConnectionPoolService.name);
   private metrics: PoolMetrics[] = [];
   private readonly maxMetricsHistory = 1000;
+  private monitoringInterval: NodeJS.Timeout | null = null;
+  private acquisitionTimes: number[] = [];
+  private readonly maxAcquisitionSamples = 500;
 
   constructor(
     private configService: ConfigService,
     private dataSource: DataSource,
-  ) {
+  ) {}
+
+  onModuleInit() {
     this.initializePoolMonitoring();
+    this.logPoolConfiguration();
+  }
+
+  onModuleDestroy() {
+    if (this.monitoringInterval) {
+      clearInterval(this.monitoringInterval);
+      this.monitoringInterval = null;
+    }
+  }
+
+  private logPoolConfiguration() {
+    const pool = this.getPool();
+    if (!pool) return;
+
+    this.logger.log('Database connection pool initialized', {
+      max: pool.options?.max,
+      min: pool.options?.min,
+      idleTimeoutMillis: pool.options?.idleTimeoutMillis,
+      connectionTimeoutMillis: pool.options?.connectionTimeoutMillis,
+    });
   }
 
   private initializePoolMonitoring() {
-    setInterval(() => {
+    const intervalMs = parseInt(
+      this.configService.get<string>('DB_POOL_MONITOR_INTERVAL') || '30000',
+      10,
+    );
+
+    this.monitoringInterval = setInterval(() => {
       this.collectMetrics();
-    }, 30000); // Collect every 30 seconds
+      this.checkPoolHealth();
+    }, intervalMs);
+  }
+
+  private getPool(): any {
+    return (this.dataSource.driver as any).pool;
   }
 
   private collectMetrics() {
     try {
-      const pool = (this.dataSource.driver as any).pool;
+      const pool = this.getPool();
       if (!pool) return;
 
+      const maxPoolSize = this.configService.get<number>('DATABASE_POOL_MAX', 20);
+      const minPoolSize = this.configService.get<number>('DATABASE_POOL_MIN', 2);
+
+      const activeConnections =
+        pool._activeConnections?.length ??
+        pool.totalCount - pool.idleCount ??
+        0;
+      const idleConnections =
+        pool._idleConnections?.length ?? pool.idleCount ?? 0;
+      const waitingRequests =
+        pool._waitingRequests?.length ?? pool.waitingCount ?? 0;
+      const totalConnections =
+        pool._allConnections?.length ?? pool.totalCount ?? activeConnections + idleConnections;
+
       const metrics: PoolMetrics = {
-        activeConnections: pool._activeConnections?.length || 0,
-        idleConnections: pool._idleConnections?.length || 0,
-        waitingRequests: pool._waitingRequests?.length || 0,
-        totalConnections: pool._allConnections?.length || 0,
+        activeConnections,
+        idleConnections,
+        waitingRequests,
+        totalConnections,
         utilizationPercentage:
-          ((pool._activeConnections?.length || 0) /
-            (pool._allConnections?.length || 1)) *
-          100,
+          totalConnections > 0 ? (activeConnections / maxPoolSize) * 100 : 0,
+        maxPoolSize,
+        minPoolSize,
         timestamp: new Date(),
       };
 
@@ -52,24 +111,41 @@ export class ConnectionPoolService {
         this.metrics.shift();
       }
 
-      // Alert on high utilization
       if (metrics.utilizationPercentage > 80) {
         this.logger.warn(
-          `High connection pool utilization: ${metrics.utilizationPercentage.toFixed(2)}%`,
+          `High connection pool utilization: ${metrics.utilizationPercentage.toFixed(2)}% (${activeConnections}/${maxPoolSize})`,
         );
       }
 
-      // Alert on waiting requests
-      if (metrics.waitingRequests > 5) {
+      if (waitingRequests > 5) {
         this.logger.warn(
-          `Connection pool queue building up: ${metrics.waitingRequests} waiting requests`,
+          `Connection pool queue building: ${waitingRequests} requests waiting`,
         );
       }
     } catch (error) {
       this.logger.error('Failed to collect pool metrics', error);
     }
   }
 
+  private async checkPoolHealth(): Promise<void> {
+    const start = Date.now();
+    try {
+      await this.dataSource.query('SELECT 1');
+      const latencyMs = Date.now() - start;
+
+      this.acquisitionTimes.push(latencyMs);
+      if (this.acquisitionTimes.length > this.maxAcquisitionSamples) {
+        this.acquisitionTimes.shift();
+      }
+
+      if (latencyMs > 500) {
+        this.logger.warn(`Slow DB health check: ${latencyMs}ms`);
+      }
+    } catch (error) {
+      this.logger.error('Pool health check failed', error);
+    }
+  }
+
   getMetrics(): PoolMetrics[] {
     return this.metrics;
   }
@@ -80,43 +156,84 @@ export class ConnectionPoolService {
       : null;
   }
 
-  getAverageUtilization(minutes: number = 5): number {
+  getAverageUtilization(minutes = 5): number {
     const cutoff = new Date(Date.now() - minutes * 60 * 1000);
     const recentMetrics = this.metrics.filter((m) => m.timestamp > cutoff);
-
     if (recentMetrics.length === 0) return 0;
-
-    const sum = recentMetrics.reduce(
-      (acc, m) => acc + m.utilizationPercentage,
-      0,
+    return (
+      recentMetrics.reduce((acc, m) => acc + m.utilizationPercentage, 0) /
+      recentMetrics.length
     );
-    return sum / recentMetrics.length;
   }
 
-  async checkPoolHealth(): Promise<boolean> {
+  async checkPoolHealth_(): Promise<boolean> {
     try {
       const result = await this.dataSource.query('SELECT 1');
       return !!result;
-    } catch (error) {
-      this.logger.error('Pool health check failed', error);
+    } catch {
       return false;
     }
   }
 
+  async getHealthStatus(): Promise<PoolHealthStatus> {
+    const start = Date.now();
+    const healthy = await this.checkPoolHealth_();
+    const latencyMs = Date.now() - start;
+    const latest = this.getLatestMetrics();
+
+    return {
+      healthy,
+      utilizationPercentage: latest?.utilizationPercentage ?? 0,
+      waitingRequests: latest?.waitingRequests ?? 0,
+      latencyMs,
+      message: healthy
+        ? `Pool healthy, ${latencyMs}ms latency`
+        : 'Pool unhealthy - connection check failed',
+    };
+  }
+
   async detectConnectionLeaks(): Promise<number> {
-    const pool = (this.dataSource.driver as any).pool;
+    const pool = this.getPool();
     if (!pool) return 0;
 
-    const activeConnections = pool._activeConnections?.length || 0;
+    const activeConnections = pool._activeConnections?.length ?? 0;
     const maxPoolSize = this.configService.get<number>('DATABASE_POOL_MAX', 20);
 
     if (activeConnections > maxPoolSize * 0.9) {
       this.logger.warn(
-        `Potential connection leak detected: ${activeConnections}/${maxPoolSize}`,
+        `Potential connection leak: ${activeConnections}/${maxPoolSize} connections active`,
       );
-      return activeConnections;
     }
 
-    return 0;
+    return activeConnections;
+  }
+
+  getConnectionAcquisitionStats() {
+    const times = this.acquisitionTimes;
+    if (times.length === 0) {
+      return { samples: 0, avgMs: 0, p95Ms: 0, p99Ms: 0, maxMs: 0 };
+    }
+
+    const sorted = [...times].sort((a, b) => a - b);
+    return {
+      samples: times.length,
+      avgMs: times.reduce((a, b) => a + b, 0) / times.length,
+      p95Ms: sorted[Math.floor(sorted.length * 0.95)] || 0,
+      p99Ms: sorted[Math.floor(sorted.length * 0.99)] || 0,
+      maxMs: sorted[sorted.length - 1] || 0,
+    };
+  }
+
+  getPoolSummary() {
+    const latest = this.getLatestMetrics();
+    const avgUtil5m = this.getAverageUtilization(5);
+    const avgUtil30m = this.getAverageUtilization(30);
+
+    return {
+      current: latest,
+      averageUtilization: { last5Minutes: avgUtil5m, last30Minutes: avgUtil30m },
+      acquisitionLatency: this.getConnectionAcquisitionStats(),
+      metricsCollected: this.metrics.length,
+    };
   }
 }

backend/src/common/database/connection-pool.controller.ts

@@ -0,0 +1,63 @@
+import { Controller, Get, Post, UseGuards } from '@nestjs/common';
+import {
+  ApiTags,
+  ApiOperation,
+  ApiResponse,
+  ApiBearerAuth,
+} from '@nestjs/swagger';
+import { ConnectionPoolService } from './connection-pool.config';
+import { ConnectionRetryService } from './connection-retry.service';
+import { JwtAuthGuard } from '../../auth/guards/jwt-auth.guard';
+
+@ApiTags('Database Pool')
+@Controller('db/pool')
+@UseGuards(JwtAuthGuard)
+@ApiBearerAuth('access-token')
+export class ConnectionPoolController {
+  constructor(
+    private readonly poolService: ConnectionPoolService,
+    private readonly retryService: ConnectionRetryService,
+  ) {}
+
+  @Get('summary')
+  @ApiOperation({ summary: 'Get connection pool summary with utilization trends' })
+  @ApiResponse({ status: 200, description: 'Pool summary including current metrics, averages, and acquisition latency' })
+  getSummary() {
+    return this.poolService.getPoolSummary();
+  }
+
+  @Get('metrics')
+  @ApiOperation({ summary: 'Get raw connection pool metrics history' })
+  getMetrics() {
+    return {
+      metrics: this.poolService.getMetrics(),
+      latest: this.poolService.getLatestMetrics(),
+    };
+  }
+
+  @Get('health')
+  @ApiOperation({ summary: 'Check connection pool health' })
+  async getHealth() {
+    return this.poolService.getHealthStatus();
+  }
+
+  @Get('leaks')
+  @ApiOperation({ summary: 'Detect potential connection leaks' })
+  async detectLeaks() {
+    const leaked = await this.poolService.detectConnectionLeaks();
+    return { suspectedLeaks: leaked };
+  }
+
+  @Post('reconnect')
+  @ApiOperation({ summary: 'Force a reconnection health check' })
+  async reconnect() {
+    const success = await this.retryService.checkAndReconnect();
+    return { success, timestamp: new Date().toISOString() };
+  }
+
+  @Get('retry-stats')
+  @ApiOperation({ summary: 'Get connection retry statistics' })
+  getRetryStats() {
+    return this.retryService.getRetryStats();
+  }
+}

backend/src/common/database/connection-pool.module.ts

@@ -1,8 +1,11 @@
 import { Module } from '@nestjs/common';
 import { ConnectionPoolService } from './connection-pool.config';
+import { ConnectionRetryService } from './connection-retry.service';
+import { ConnectionPoolController } from './connection-pool.controller';
 
 @Module({
-  providers: [ConnectionPoolService],
-  exports: [ConnectionPoolService],
+  controllers: [ConnectionPoolController],
+  providers: [ConnectionPoolService, ConnectionRetryService],
+  exports: [ConnectionPoolService, ConnectionRetryService],
 })
 export class ConnectionPoolModule {}