diff --git a/.aiignore b/.aiignore new file mode 100644 index 00000000..58063d8b --- /dev/null +++ b/.aiignore @@ -0,0 +1,4 @@ +.env +.env.* +keys/ +secrets.json \ No newline at end of file diff --git a/.env.example b/.env.example index 7b1ba7ae..1b561ea0 100644 --- a/.env.example +++ b/.env.example @@ -53,3 +53,9 @@ RAZORPAY_KEY_ID=rzp_testx_xxxx RAZORPAY_KEY_SECRET=xxxx RAZORPAY_PLAN_ID=xxxx RAZORPAY_WEBHOOK_SECRET=xxxx + +# ── Internal Microservices (AI & Python) ────────────────────────────────────── +# Required for AI Query Builder and Python microservice communication +PYTHON_SERVICE_URL=http://localhost:8000 +INTERNAL_SECRET=generate_a_random_32_char_secret_here +GROQ_API_KEY=gsk_your_groq_api_key_here diff --git a/.github/workflows/weekly-changelog.yml b/.github/workflows/weekly-changelog.yml index dc917bd3..0c02c33d 100644 --- a/.github/workflows/weekly-changelog.yml +++ b/.github/workflows/weekly-changelog.yml @@ -72,13 +72,11 @@ jobs: # Create monthly file if it doesn't exist yet if [ ! -f "$FILE" ]; then - cat > "$FILE" << MDXEOF ---- -title: Changelog -description: What's new in urBackend — new features, improvements, and fixes. ---- - -MDXEOF + echo "---" > "$FILE" + echo "title: Changelog" >> "$FILE" + echo "description: What's new in urBackend — new features, improvements, and fixes." >> "$FILE" + echo "---" >> "$FILE" + echo "" >> "$FILE" fi # Insert new entry right after the closing --- of frontmatter diff --git a/.gitignore b/.gitignore index c380342b..117235b7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,12 @@ coverage/ .turbo/ plan.md /.kiroo -/xtemp/ \ No newline at end of file +/xtemp/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +venv/ +env/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7a73a41b..3b664107 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,2 +1,3 @@ { + "git.ignoreLimitWarning": true } \ No newline at end of file diff --git a/apps/dashboard-api/src/__tests__/internalPythonClient.test.js b/apps/dashboard-api/src/__tests__/internalPythonClient.test.js new file mode 100644 index 00000000..32c81b7a --- /dev/null +++ b/apps/dashboard-api/src/__tests__/internalPythonClient.test.js @@ -0,0 +1,62 @@ +const axios = require('axios'); +const crypto = require('crypto'); +const { forwardToPythonService } = require('../utils/internalPythonClient'); + +describe('internalPythonClient', () => { + let originalEnv; + + beforeEach(() => { + originalEnv = process.env; + process.env = { ...originalEnv }; + jest.clearAllMocks(); + + // Mock Date.now to freeze timestamp for assertions + jest.spyOn(Date, 'now').mockImplementation(() => 1609459200000); // 2021-01-01T00:00:00.000Z + jest.spyOn(axios, 'post').mockResolvedValue({ data: { success: true } }); + }); + + afterEach(() => { + process.env = originalEnv; + jest.restoreAllMocks(); + }); + + test('throws error if INTERNAL_SECRET is missing', async () => { + delete process.env.INTERNAL_SECRET; + + await expect(forwardToPythonService('/test', {})) + .rejects + .toThrow("INTERNAL_SECRET is not defined in environment"); + }); + + test('generates correct HMAC signature and calls axios', async () => { + process.env.INTERNAL_SECRET = 'test-secret'; + process.env.PYTHON_SERVICE_URL = 'http://test-python.local'; + + const path = '/ai/query-builder'; + const payload = { prompt: "test prompt" }; + const payloadString = JSON.stringify(payload); + const timestamp = "1609459200000"; + + // Calculate expected signature manually + const expectedSignature = crypto + .createHmac('sha256', 'test-secret') + .update(`${timestamp}.${payloadString}`) + .digest('hex'); + + const result = await forwardToPythonService(path, payload); + + expect(axios.post).toHaveBeenCalledTimes(1); + expect(axios.post).toHaveBeenCalledWith( + `http://test-python.local${path}`, + payloadString, + { + headers: { + 'X-Internal-Signature': expectedSignature, + 'X-Timestamp': timestamp, + 'Content-Type': 'application/json' + } + } + ); + expect(result).toEqual({ success: true }); + }); +}); diff --git a/apps/dashboard-api/src/app.js b/apps/dashboard-api/src/app.js index 64db3478..185a6c04 100644 --- a/apps/dashboard-api/src/app.js +++ b/apps/dashboard-api/src/app.js @@ -105,9 +105,11 @@ const analyticsRoute = require('./routes/analytics'); const billingRoute = require('./routes/billing'); const eventsRoute = require('./routes/events'); const adminMetricsRoute = require('./routes/admin.metrics'); +const aiRoute = require('./routes/ai.routes'); app.use('/api/auth', authRoute); app.use('/api/projects', dashboardLimiter, projectRoute); +app.use('/api/projects/:projectId/ai', dashboardLimiter, aiRoute); app.use('/api/projects', dashboardLimiter, webhookRoute); app.use('/api/releases', releaseRoute); app.use('/api/analytics', dashboardLimiter, analyticsRoute); diff --git a/apps/dashboard-api/src/controllers/ai.controller.js b/apps/dashboard-api/src/controllers/ai.controller.js new file mode 100644 index 00000000..9437ce39 --- /dev/null +++ b/apps/dashboard-api/src/controllers/ai.controller.js @@ -0,0 +1,124 @@ +const { Project } = require('@urbackend/common/src/models'); +const { forwardToPythonService } = require('../utils/internalPythonClient'); +const { AppError } = require('@urbackend/common'); + +/** + * Controller to handle AI Query Builder requests. + */ +const queryBuilder = async (req, res, next) => { + try { + const { projectId } = req.params; + const { collectionName, prompt } = req.body; + + const mongoose = require('mongoose'); + if (!mongoose.Types.ObjectId.isValid(projectId)) { + throw new AppError(400, "Invalid project ID"); + } + + if (typeof collectionName !== 'string' || typeof prompt !== 'string') { + throw new AppError(400, "Collection name and prompt must be strings"); + } + + const safeCollectionName = collectionName.trim(); + const safePrompt = prompt.trim(); + + if (!safeCollectionName || !safePrompt) { + throw new AppError(400, "Collection name and prompt are required"); + } + + if (safeCollectionName === 'users') { + throw new AppError(403, "Cannot query the users collection via AI"); + } + + // 1. Fetch the project and specifically the requested collection schema + const project = await Project.findOne( + { _id: projectId, owner: req.user._id, "collections.name": safeCollectionName }, + { "collections.$": 1 } + ); + + if (!project || !project.collections || project.collections.length === 0) { + throw new AppError(404, "Collection not found or access denied"); + } + + const collection = project.collections[0]; + const allowedFields = new Set([ + ...collection.model.map(field => field.key), + '_id', + 'createdAt', + 'updatedAt' + ]); + + // 2. Extract simplified schema fields for the LLM + // We only send key and type to save tokens and prevent confusion + const schemaFields = collection.model.map(field => ({ + key: field.key, + type: field.type + })); + + // Add implicit MongoDB fields + schemaFields.push( + { key: "_id", type: "OBJECTID" }, + { key: "createdAt", type: "DATE" }, + { key: "updatedAt", type: "DATE" } + ); + + // 3. Forward request to Python Service + const aiResponse = await forwardToPythonService('/ai/query-builder', { + prompt: safePrompt, + schema_fields: schemaFields + }); + + // 4. Return the structured JSON to the frontend + // Ensure filters is always an array to prevent frontend crash + const rawFilters = Array.isArray(aiResponse.filters) ? aiResponse.filters : []; + const allowedOperators = new Set(['=', '_gt', '_lt', '_gte', '_lte', '_ne', '_regex']); + const safeFilters = rawFilters.filter(f => { + const isPrimitiveValue = ['string', 'number', 'boolean'].includes(typeof f?.value); + return ( + f && + typeof f.field === 'string' && + typeof f.operator === 'string' && + allowedFields.has(f.field) && + allowedOperators.has(f.operator) && + isPrimitiveValue + ); + }); + + res.status(200).json({ + success: true, + data: { + filters: safeFilters, + sort: typeof aiResponse.sort === 'string' ? aiResponse.sort : '-createdAt' + }, + message: "Query built successfully" + }); + + } catch (error) { + // Forward expected AppErrors + if (error instanceof AppError) { + return next(error); + } + + // Wrap Python/Axios errors + if (error.response && error.response.data) { + console.error("AI Service returned error:", error.response.status, error.response.data); + + let errorMessage = "AI Service Error"; + if (typeof error.response.data === 'string') { + errorMessage = error.response.data; + } else if (error.response.data.detail) { + errorMessage = typeof error.response.data.detail === 'string' ? error.response.data.detail : JSON.stringify(error.response.data.detail); + } else { + errorMessage = JSON.stringify(error.response.data); + } + + return next(new AppError(error.response.status || 500, errorMessage)); + } + + next(new AppError(500, "Failed to build query via AI")); + } +}; + +module.exports = { + queryBuilder +}; diff --git a/apps/dashboard-api/src/routes/ai.routes.js b/apps/dashboard-api/src/routes/ai.routes.js new file mode 100644 index 00000000..29b03347 --- /dev/null +++ b/apps/dashboard-api/src/routes/ai.routes.js @@ -0,0 +1,16 @@ +const express = require('express'); +const router = express.Router({ mergeParams: true }); // mergeParams is crucial to access :projectId +const aiController = require('../controllers/ai.controller'); +const authMiddleware = require('../middlewares/authMiddleware'); + +// All AI routes require the user to be authenticated +router.use(authMiddleware); + +/** + * @route POST /api/projects/:projectId/ai/query-builder + * @desc Generate MongoDB filters from natural language + * @access Private + */ +router.post('/query-builder', aiController.queryBuilder); + +module.exports = router; diff --git a/apps/dashboard-api/src/utils/internalPythonClient.js b/apps/dashboard-api/src/utils/internalPythonClient.js new file mode 100644 index 00000000..f38c132e --- /dev/null +++ b/apps/dashboard-api/src/utils/internalPythonClient.js @@ -0,0 +1,42 @@ +const crypto = require('crypto'); +const axios = require('axios'); + +/** + * Forwards a request to the internal Python microservice with HMAC-SHA256 signature. + * @param {string} path - The path on the Python service (e.g., "/ai/query-builder") + * @param {object} payload - The JSON payload to send + * @returns {Promise} The response data from Python service + */ +const forwardToPythonService = async (path, payload) => { + const pythonUrl = process.env.PYTHON_SERVICE_URL || 'http://localhost:8000'; + const secret = process.env.INTERNAL_SECRET; + + if (!secret) { + throw new Error("INTERNAL_SECRET is not defined in environment"); + } + + const payloadString = JSON.stringify(payload); + const timestamp = Date.now().toString(); + + // Generate HMAC-SHA256 signature + const signature = crypto + .createHmac('sha256', secret) + .update(`${timestamp}.${payloadString}`) + .digest('hex'); + + try { + const response = await axios.post(`${pythonUrl}${path}`, payloadString, { + headers: { + 'X-Internal-Signature': signature, + 'X-Timestamp': timestamp, + 'Content-Type': 'application/json' + } + }); + return response.data; + } catch (error) { + console.error("Error communicating with Python Service:", error.response?.data || error.message); + throw error; + } +}; + +module.exports = { forwardToPythonService }; diff --git a/apps/dashboard-api/test_python.js b/apps/dashboard-api/test_python.js new file mode 100644 index 00000000..734279af --- /dev/null +++ b/apps/dashboard-api/test_python.js @@ -0,0 +1,37 @@ +const axios = require('axios'); +const crypto = require('crypto'); + +const run = async () => { + const timestamp = Date.now().toString(); + const payload = JSON.stringify({ prompt: "test", schema_fields: [] }); + + const secret = process.env.TEST_SECRET || process.env.INTERNAL_SECRET; + const apiUrl = process.env.AI_API_URL || 'http://127.0.0.1:8000/ai/query-builder'; + + if (!secret) { + console.error("Missing TEST_SECRET or INTERNAL_SECRET in environment variables."); + process.exit(1); + } + + const signature = crypto.createHmac('sha256', secret).update(`${timestamp}.${payload}`).digest('hex'); + + try { + const res = await axios.post(apiUrl, payload, { + headers: { + 'X-Internal-Signature': signature, + 'X-Timestamp': timestamp, + 'Content-Type': 'application/json' + } + }); + console.log("Success:", res.data); + } catch (e) { + if (e.response) { + console.log("Error status:", e.response.status); + console.log("Error data:", e.response.data); + } else { + console.log("No response:", e.message); + } + } +}; + +run(); diff --git a/apps/python-service/.env.example b/apps/python-service/.env.example new file mode 100644 index 00000000..6e0eb3da --- /dev/null +++ b/apps/python-service/.env.example @@ -0,0 +1,12 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Python Microservice — Environment Variables +# Copy this file to .env and fill in your values +# ───────────────────────────────────────────────────────────────────────────── + +# ── Security ───────────────────────────────────────────────────────────────── +# Must match the INTERNAL_SECRET in the root Dashboard API .env +INTERNAL_SECRET=generate_a_random_32_char_secret_here + +# ── AI APIs ────────────────────────────────────────────────────────────────── +# Required for AI Query Builder +GROQ_API_KEY=gsk_your_groq_api_key_here diff --git a/apps/python-service/config.py b/apps/python-service/config.py new file mode 100644 index 00000000..387b2f0a --- /dev/null +++ b/apps/python-service/config.py @@ -0,0 +1,11 @@ +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + +class Settings(BaseSettings): + INTERNAL_SECRET: str = Field(min_length=32) + GROQ_API_KEY: str = "" # Default empty, can be supplied via BYOK later + REDIS_URL: str = "redis://localhost:6379" + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") + +settings = Settings() diff --git a/apps/python-service/dependencies.py b/apps/python-service/dependencies.py new file mode 100644 index 00000000..f973c5f4 --- /dev/null +++ b/apps/python-service/dependencies.py @@ -0,0 +1,45 @@ +import hmac +import hashlib +import time +import redis.asyncio as redis +from fastapi import Request, HTTPException, Depends +from config import settings + +# Initialize Redis connection +redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True) + +async def verify_signature(request: Request): + signature = request.headers.get("X-Internal-Signature") + timestamp_str = request.headers.get("X-Timestamp") + + if not signature or not timestamp_str: + raise HTTPException(status_code=403, detail="Missing authentication headers") + + try: + timestamp = int(timestamp_str) + except ValueError as e: + raise HTTPException(status_code=403, detail="Invalid timestamp format") from e + + # Replay attack protection — 30s window + current_time_ms = int(time.time() * 1000) + if abs(current_time_ms - timestamp) > 30000: + raise HTTPException(status_code=403, detail="Request expired") + + # Read body to verify signature + body_bytes = await request.body() + payload = body_bytes.decode('utf-8') + + expected_mac = hmac.new( + settings.INTERNAL_SECRET.encode('utf-8'), + f"{timestamp}.{payload}".encode('utf-8'), + hashlib.sha256 + ).hexdigest() + + if not hmac.compare_digest(expected_mac, signature): + raise HTTPException(status_code=403, detail="Invalid signature") + + # Redis Nonce check for exact replay attack prevention (Atomic SET NX) + nonce_key = f"internal:nonce:{timestamp}:{signature}" + success = await redis_client.set(nonce_key, "1", ex=30, nx=True) + if not success: + raise HTTPException(status_code=403, detail="Replay attack detected") diff --git a/apps/python-service/main.py b/apps/python-service/main.py new file mode 100644 index 00000000..ebf3ee41 --- /dev/null +++ b/apps/python-service/main.py @@ -0,0 +1,10 @@ +from fastapi import FastAPI +app = FastAPI(title="urBackend Python Service") + +@app.get("/health") +async def health_check(): + return {"status": "ok"} + +from routers import ai + +app.include_router(ai.router) diff --git a/apps/python-service/requirements.txt b/apps/python-service/requirements.txt new file mode 100644 index 00000000..c69a2573 --- /dev/null +++ b/apps/python-service/requirements.txt @@ -0,0 +1,8 @@ +fastapi==0.110.1 +uvicorn[standard]==0.29.0 +langchain-groq==1.1.2 +langsmith==0.8.5 +pydantic==2.7.4 +pydantic-settings==2.2.1 +python-dotenv==1.0.1 +redis==5.0.3 diff --git a/apps/python-service/routers/ai.py b/apps/python-service/routers/ai.py new file mode 100644 index 00000000..7f28e33a --- /dev/null +++ b/apps/python-service/routers/ai.py @@ -0,0 +1,77 @@ +import asyncio +import logging +from fastapi import APIRouter, HTTPException, Depends +from pydantic import BaseModel, Field +from typing import List, Union +from langchain_groq import ChatGroq +from langchain_core.prompts import ChatPromptTemplate +from config import settings +from dependencies import verify_signature + +router = APIRouter(prefix="/ai", tags=["ai"], dependencies=[Depends(verify_signature)]) +logger = logging.getLogger(__name__) + +class FilterItem(BaseModel): + field: str = Field(description="The exact field name from the schema") + operator: str = Field(description="One of: '=', '_gt', '_lt', '_gte', '_lte', '_ne', '_regex'") + value: Union[str, int, float, bool] = Field(description="The value to filter by") + +class QueryResult(BaseModel): + filters: List[FilterItem] = Field(default_factory=list, description="List of MongoDB filters to apply to the frontend") + sort: str = Field(default="-createdAt", description="MongoDB sort string, e.g. '-createdAt' or 'name'. Default to '-createdAt'") + +class QueryBuilderRequest(BaseModel): + prompt: str + schema_fields: List[dict] + +@router.post("/query-builder", response_model=QueryResult) +async def query_builder(request: QueryBuilderRequest): + if not settings.GROQ_API_KEY: + raise HTTPException(status_code=500, detail="Groq API key not configured") + + try: + # Initialize Groq LLM + llm = ChatGroq(api_key=settings.GROQ_API_KEY, model_name="llama-3.1-8b-instant", temperature=0) + + # Enforce structured output based on our Pydantic schema + structured_llm = llm.with_structured_output(QueryResult) + + # Build the system prompt + system_prompt = """You are a highly intelligent database query builder for a MongoDB-based BaaS called urBackend. +Your job is to take the user's natural language request and convert it into a set of structured filters and a sort string. +You will be provided with the exact schema of the collection. You MUST ONLY use the fields defined in the schema. +Do NOT hallucinate fields that do not exist. +If the user's prompt is vague, use your best judgement based on the available schema fields. + +CRITICAL INSTRUCTION: +Your `filters` output MUST be a list (array) of objects matching the FilterItem schema exactly (each having `field`, `operator`, and `value`). +DO NOT output a raw MongoDB filter dict like `{{"price": {{"$gt": 1000}} }}`. +Correct format: `[{{ "field": "price", "operator": "_gt", "value": 1000 }}]`. + +Schema Fields: {schema}""" + + prompt = ChatPromptTemplate.from_messages([ + ("system", system_prompt), + ("human", "{user_prompt}") + ]) + + # Create the LangChain chain + chain = prompt | structured_llm + + # Invoke the chain with a timeout to prevent hanging requests + result = await asyncio.wait_for( + chain.ainvoke({ + "schema": str(request.schema_fields), + "user_prompt": request.prompt + }), + timeout=15.0 + ) + + return result + + except asyncio.TimeoutError as e: + logger.error("AI Query Builder timeout", exc_info=True) + raise HTTPException(status_code=504, detail="AI request timed out") from e + except Exception as e: + logger.error("AI Query Builder failed", exc_info=True) + raise HTTPException(status_code=500, detail="Internal server error") from e diff --git a/apps/python-service/tests/test_auth.py b/apps/python-service/tests/test_auth.py new file mode 100644 index 00000000..c017b651 --- /dev/null +++ b/apps/python-service/tests/test_auth.py @@ -0,0 +1,113 @@ +import pytest +import hmac +import hashlib +import time +import os +import sys +from pathlib import Path +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient +from unittest.mock import AsyncMock, patch + +SERVICE_ROOT = Path(__file__).resolve().parents[1] +if str(SERVICE_ROOT) not in sys.path: + sys.path.insert(0, str(SERVICE_ROOT)) +os.environ.setdefault("INTERNAL_SECRET", "test-internal-secret") + +from dependencies import verify_signature +from config import settings +from fastapi import Depends + +# Create a dummy FastAPI app to test the dependency +app = FastAPI() + +@app.post("/test-endpoint", dependencies=[Depends(verify_signature)]) +async def dummy_endpoint(request: Request): + return {"message": "success"} + +client = TestClient(app) + +def generate_valid_signature(timestamp: int, payload: str) -> str: + return hmac.new( + settings.INTERNAL_SECRET.encode('utf-8'), + f"{timestamp}.{payload}".encode('utf-8'), + hashlib.sha256 + ).hexdigest() + +@pytest.fixture(autouse=True) +def mock_redis(): + with patch('dependencies.redis_client.get', new_callable=AsyncMock) as mock_get, \ + patch('dependencies.redis_client.set', new_callable=AsyncMock) as mock_set: + mock_set.return_value = True # Simulate nonce does not exist (SET NX succeeds) + yield mock_get, mock_set + +def test_missing_headers(): + response = client.post("/test-endpoint", json={"foo": "bar"}) + assert response.status_code == 403 + assert response.json()["detail"] == "Missing authentication headers" + +def test_invalid_signature(): + timestamp = int(time.time() * 1000) + response = client.post( + "/test-endpoint", + json={"foo": "bar"}, + headers={ + "X-Timestamp": str(timestamp), + "X-Internal-Signature": "invalid_signature_hex" + } + ) + assert response.status_code == 403 + assert response.json()["detail"] == "Invalid signature" + +def test_expired_timestamp(): + # 60 seconds in the past + timestamp = int(time.time() * 1000) - 60000 + payload = '{"foo": "bar"}' + signature = generate_valid_signature(timestamp, payload) + + response = client.post( + "/test-endpoint", + content=payload, + headers={ + "X-Timestamp": str(timestamp), + "X-Internal-Signature": signature + } + ) + assert response.status_code == 403 + assert response.json()["detail"] == "Request expired" + +def test_valid_request(): + timestamp = int(time.time() * 1000) + payload = '{"foo": "bar"}' + signature = generate_valid_signature(timestamp, payload) + + response = client.post( + "/test-endpoint", + content=payload, + headers={ + "X-Timestamp": str(timestamp), + "X-Internal-Signature": signature + } + ) + assert response.status_code == 200 + assert response.json()["message"] == "success" + +def test_replay_attack(mock_redis): + _, mock_set = mock_redis + # Simulate that Redis says this nonce already exists (SET NX fails) + mock_set.return_value = None + + timestamp = int(time.time() * 1000) + payload = '{"foo": "bar"}' + signature = generate_valid_signature(timestamp, payload) + + response = client.post( + "/test-endpoint", + content=payload, + headers={ + "X-Timestamp": str(timestamp), + "X-Internal-Signature": signature + } + ) + assert response.status_code == 403 + assert response.json()["detail"] == "Replay attack detected" diff --git a/apps/web-dashboard/src/components/Database/AiQueryBar.jsx b/apps/web-dashboard/src/components/Database/AiQueryBar.jsx new file mode 100644 index 00000000..1f5a5eee --- /dev/null +++ b/apps/web-dashboard/src/components/Database/AiQueryBar.jsx @@ -0,0 +1,90 @@ +import React, { useState } from 'react'; +import { Sparkles, Loader2 } from 'lucide-react'; +import toast from 'react-hot-toast'; +import api from '../../utils/api'; + +const AiQueryBar = ({ projectId, activeCollection, onFiltersGenerated }) => { + const [prompt, setPrompt] = useState(''); + const [isLoading, setIsLoading] = useState(false); + const canSubmit = Boolean(projectId && activeCollection?.name); + + const handleSubmit = async (e) => { + e.preventDefault(); + if (!prompt.trim() || !canSubmit) return; + + setIsLoading(true); + try { + const res = await api.post(`/api/projects/${projectId}/ai/query-builder`, { + collectionName: activeCollection?.name, + prompt: prompt.trim() + }); + + if (res.data?.success && res.data?.data) { + const { filters, sort } = res.data.data; + if (typeof onFiltersGenerated === 'function') { + onFiltersGenerated(filters, sort); + } + toast.success('AI query applied!'); + setPrompt(''); + } else { + toast.error('Failed to generate query.'); + } + } catch (error) { + console.error('AI Query Error:', error); + toast.error(error.response?.data?.message || 'Error communicating with AI service'); + } finally { + setIsLoading(false); + } + }; + + return ( +
+ + setPrompt(e.target.value)} + placeholder="Ask AI to filter data..." + aria-label="Ask AI to filter data" + disabled={isLoading || !activeCollection || !projectId} + style={{ + background: 'transparent', + border: 'none', + color: '#fff', + flex: 1, + fontSize: '0.85rem', + outline: 'none' + }} + /> + {isLoading && } + + + ); +}; + +export default AiQueryBar; diff --git a/apps/web-dashboard/src/components/Database/DatabaseFilter.jsx b/apps/web-dashboard/src/components/Database/DatabaseFilter.jsx index a03cc3f5..0ce0449e 100644 --- a/apps/web-dashboard/src/components/Database/DatabaseFilter.jsx +++ b/apps/web-dashboard/src/components/Database/DatabaseFilter.jsx @@ -106,6 +106,9 @@ const DatabaseFilter = ({ style={{ width: '35%', height: '30px', padding: '0 6px', fontSize: '0.7rem' }} > + + + {activeCollection?.model?.map(f => ( ))} diff --git a/apps/web-dashboard/src/components/Database/DatabaseHeader.jsx b/apps/web-dashboard/src/components/Database/DatabaseHeader.jsx index bf3cfbb4..4ce55a3c 100644 --- a/apps/web-dashboard/src/components/Database/DatabaseHeader.jsx +++ b/apps/web-dashboard/src/components/Database/DatabaseHeader.jsx @@ -3,12 +3,13 @@ import { Menu, List as ListIcon, Table as TableIcon, Code, Filter, RefreshCw, Shield, Plus } from 'lucide-react'; +import AiQueryBar from './AiQueryBar'; const DatabaseHeader = ({ project, activeCollection, dataLength, viewMode, setViewMode, showFilterMenu, setShowFilterMenu, filtersCount, onRefresh, onRlsClick, onAddRecord, onOpenSidebar, - showDeleted, setShowDeleted + showDeleted, setShowDeleted, onFiltersGenerated }) => { return (
+ {activeCollection?.name !== 'users' && ( +
+ +
+ )} {dataLength} Records {/* Soft Delete Toggle */} diff --git a/apps/web-dashboard/src/pages/Database.jsx b/apps/web-dashboard/src/pages/Database.jsx index 3a1b63c8..f80c0626 100644 --- a/apps/web-dashboard/src/pages/Database.jsx +++ b/apps/web-dashboard/src/pages/Database.jsx @@ -156,6 +156,15 @@ export default function Database() { } catch { toast.error("Failed to delete document"); } }; + const handleFiltersGenerated = (aiFilters, aiSort) => { + setQueryParams(prev => ({ + ...prev, + page: 1, + filters: aiFilters, + sort: aiSort || prev.sort + })); + }; + /** * Generates an RLS-aware cURL snippet for the active collection. * Uses the secret key if RLS is disabled, or the publishable key with a JWT if RLS is enabled. @@ -207,6 +216,7 @@ export default function Database() { onOpenSidebar={() => setIsSidebarOpen(true)} showDeleted={showDeleted} setShowDeleted={setShowDeleted} + onFiltersGenerated={handleFiltersGenerated} />
diff --git a/docs/ISSUES/project-rate-limiting.md b/docs/ISSUES/project-rate-limiting.md new file mode 100644 index 00000000..fe81e0e9 --- /dev/null +++ b/docs/ISSUES/project-rate-limiting.md @@ -0,0 +1,77 @@ +--- +title: Project & API-key Rate Limiting +labels: bug, security, infra +assignees: '' +--- + +# Project & API-key Rate Limiting + +Date: 2026-05-22 + +## Summary + +There is confusion and a small gap in rate limiting for `apps/public-api`. + +- A global IP-based express limiter exists and is applied (`api_usage.js`). +- Per-project plan limits are enforced asynchronously via Redis in `usageGate.checkUsageLimits` after API-key verification. +- A `projectRateLimiter` express middleware exists (`projectRateLimiter.js`) but is not wired into any routes and contains an implementation issue. + +This issue tracks fixing and hardening per-project / per-API-key rate limiting so generated CRUD endpoints cannot be abused to cause DDoS or runaway DB costs. + +## Impact + +- Possible API abuse or DoS by hitting generated endpoints. +- Runaway MongoDB Atlas costs for projects using BYO-Database due to unbounded request volume. + +## Reproduction + +1. Observe that `/api/data/*` and many `/api/mail/*` endpoints accept `x-api-key`. +2. Send high-volume requests from multiple IPs or with the same API key. +3. Global IP limiter will throttle per-IP, but per-project hard limits are not applied at the express middleware layer. + +## Root causes / gaps + +- `projectRateLimiter.js` uses a nonstandard option and is never imported/used. +- `checkUsageLimits` enforces plan-based limits via Redis but is asynchronous and may allow short bursts before counters are incremented. +- Some `mail` admin routes use `verifyApiKey` + `requireSecretKey` but lack `checkUsageLimits`. + +## Proposed fix + +1. Fix `projectRateLimiter` implementation: + - Use `express-rate-limit` `max` and `keyGenerator` correctly. + - Use a Redis-backed store (e.g., `rate-limit-redis`) for production clustering. + - Keying options: prefer `req.project._id` or `req.hashedApiKey` (decide per-route). + +2. Wire `projectRateLimiter` after `verifyApiKey` on per-project routes: + - `app.use('/api/data', ..., verifyApiKey, projectRateLimiter, ...)` or add to route-level middleware arrays in `routes/data.js` and `routes/mail.js` where appropriate. + +3. Ensure `checkUsageLimits` is applied to all endpoints that should be metered (add to missing `mail` routes). + +4. (Optional) Add a per-API-key express limiter (key by `req.hashedApiKey`) for stricter per-key bursts. + +5. Add tests: unit/integration tests to validate that per-project limits reject requests with 429, and that `checkUsageLimits` and express limiter co-exist. + +6. Update docs: README / security docs to describe global, per-project and per-key limits and expected defaults. + +## Implementation checklist + +- [ ] Fix `projectRateLimiter.js` implementation (use `max`, `keyGenerator`, Redis store) +- [ ] Add `projectRateLimiter` to `routes/data.js` after `verifyApiKey` +- [ ] Add `projectRateLimiter` to `routes/mail.js` for admin/send endpoints where `requireSecretKey` is used +- [ ] Ensure missing `checkUsageLimits` calls are added to mail routes that should be metered +- [ ] Add integration tests in `apps/public-api/__tests__` that assert 429 behavior +- [ ] Update docs: `docs/ISSUES/project-rate-limiting.md` and README sections +- [ ] Optional: add per-API-key limiter and configuration flag (on by default) + +## Acceptance criteria + +- Per-project express limiter is active and returns 429 when a project's configured hard-limit is exceeded. +- Redis-backed plan checks in `checkUsageLimits` remain and are consistent with express limiter thresholds. +- Tests cover both express limiter and Redis plan enforcement. +- Documentation updated to reflect protections. + +## Notes + +- This was initially reported as “no per-project or per-API-key rate limiting” — the codebase has protections, but the `projectRateLimiter` was present and unused. The fix provides defense-in-depth (fast-fail middleware + Redis metering). + +---