Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changeset/agent-monitoring-controls.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
---

Agent monitoring controls: User-Agent headers on all automated outbound requests, outbound request log with dashboard visibility, and owner controls to pause or adjust check frequency.
540 changes: 283 additions & 257 deletions package-lock.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"check:images": "bash scripts/check-image-quality.sh"
},
"dependencies": {
"@adcp/client": "^4.19.0",
"@adcp/client": "^4.22.0",
"@anthropic-ai/sdk": "^0.82.0",
"@asteasolutions/zod-to-openapi": "^8.5.0",
"@google/generative-ai": "^0.24.1",
Expand Down
150 changes: 150 additions & 0 deletions server/public/dashboard-agents.html
Original file line number Diff line number Diff line change
Expand Up @@ -626,16 +626,36 @@ <h1>Agents</h1>
${clickableTrackPills ? '<div class="agent-tracks">' + clickableTrackPills + '</div>' : ''}
<div class="agent-track-detail" id="${cardId}-track-detail" style="display:none;"></div>
${sparkline}
<div class="agent-monitoring-controls" style="display: flex; align-items: center; gap: var(--space-3); padding: var(--space-2) 0; border-top: 1px solid var(--border-subtle); margin-top: var(--space-3);">
<label class="agent-toggle" title="Pause automated compliance and health checks">
<input type="checkbox" class="monitoring-pause-toggle" data-agent-url="${escapeHtml(agent.url)}" ${cs.monitoring_paused ? 'checked' : ''}>
<span class="agent-toggle-label">Pause automated checks</span>
</label>
<label style="display: flex; align-items: center; gap: var(--space-1); font-size: var(--text-sm); color: var(--text-secondary);">
Check every
<select class="monitoring-interval-select" data-agent-url="${escapeHtml(agent.url)}" ${cs.monitoring_paused ? 'disabled' : ''} style="font-size: var(--text-sm); padding: var(--space-1) var(--space-2); border-radius: var(--radius-sm, 4px); border: 1px solid var(--border-subtle);${cs.monitoring_paused ? ' opacity: 0.4;' : ''}">
<option value="6" ${(cs.check_interval_hours || 12) === 6 ? 'selected' : ''}>6h</option>
<option value="12" ${(cs.check_interval_hours || 12) === 12 ? 'selected' : ''}>12h</option>
<option value="24" ${(cs.check_interval_hours || 12) === 24 ? 'selected' : ''}>24h</option>
<option value="48" ${(cs.check_interval_hours || 12) === 48 ? 'selected' : ''}>48h</option>
<option value="72" ${(cs.check_interval_hours || 12) === 72 ? 'selected' : ''}>72h</option>
<option value="168" ${(cs.check_interval_hours || 12) === 168 ? 'selected' : ''}>Weekly</option>
</select>
</label>
${cs.monitoring_paused ? '<span style="font-size: var(--text-xs); color: var(--color-warning-700, #b45309);">Compliance status will not update while paused</span>' : ''}
</div>
<div class="agent-meta-row">
<span>Last checked: ${escapeHtml(lastChecked)}</span>
<div style="display: flex; gap: var(--space-3); align-items: center;">
${!hasAuth ? '<a href="/chat?prompt=' + encodeURIComponent('Connect my agent at ' + agent.url + ' for compliance monitoring.') + '" class="agent-action-link" title="Connect through Addie to save credentials">Connect agent</a>' : ''}
<button class="agent-storyboard-btn" data-agent-url="${escapeHtml(agent.url)}" data-card-id="${cardId}">Test your agent</button>
<button class="agent-history-btn" data-agent-url="${escapeHtml(agent.url)}" data-card-id="${cardId}">View history</button>
<button class="agent-requests-btn" data-agent-url="${escapeHtml(agent.url)}" data-card-id="${cardId}">View requests</button>
</div>
</div>
<div class="agent-storyboard-panel" id="${cardId}-storyboard" style="display:none;"></div>
<div class="agent-history-panel" id="${cardId}-history" style="display:none;"></div>
<div class="agent-requests-panel" id="${cardId}-requests" style="display:none;"></div>
</div>
`;
}).join('');
Expand Down Expand Up @@ -673,6 +693,136 @@ <h1>Agents</h1>
}
});

// Monitoring pause toggle
document.addEventListener('change', async function(e) {
const toggle = e.target.closest('.monitoring-pause-toggle');
if (toggle) {
const agentUrl = toggle.dataset.agentUrl;
const controlsRow = toggle.closest('.agent-monitoring-controls');
const intervalSelect = controlsRow ? controlsRow.querySelector('.monitoring-interval-select') : null;

try {
const res = await fetch(`/api/registry/agents/${encodeURIComponent(agentUrl)}/monitoring/pause`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ paused: toggle.checked }),
});
if (!res.ok) {
toggle.checked = !toggle.checked;
} else {
// Update interval dropdown disabled state
if (intervalSelect) {
intervalSelect.disabled = toggle.checked;
intervalSelect.style.opacity = toggle.checked ? '0.4' : '1';
}
// Show/hide pause warning
if (controlsRow) {
let warning = controlsRow.querySelector('.pause-warning');
if (toggle.checked && !warning) {
warning = document.createElement('span');
warning.className = 'pause-warning';
warning.style.cssText = 'font-size: var(--text-xs); color: var(--color-warning-700, #b45309);';
warning.textContent = 'Compliance status will not update while paused';
controlsRow.appendChild(warning);
} else if (!toggle.checked && warning) {
warning.remove();
}
}
}
} catch {
toggle.checked = !toggle.checked;
}
}
});

// Capture initial value for interval selects on focus (before change fires)
document.addEventListener('focus', function(e) {
const select = e.target.closest('.monitoring-interval-select');
if (select) {
select.dataset.previousValue = select.value;
}
}, true);

// Monitoring interval change
document.addEventListener('change', async function(e) {
const select = e.target.closest('.monitoring-interval-select');
if (select) {
const agentUrl = select.dataset.agentUrl;
const previousValue = select.dataset.previousValue;
select.dataset.previousValue = select.value;
try {
const res = await fetch(`/api/registry/agents/${encodeURIComponent(agentUrl)}/monitoring/interval`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ interval_hours: parseInt(select.value, 10) }),
});
if (!res.ok) {
select.value = previousValue;
}
} catch {
select.value = previousValue;
}
}
});

// View outbound requests log
document.addEventListener('click', async function(e) {
const btn = e.target.closest('.agent-requests-btn');
if (!btn) return;

const agentUrl = btn.dataset.agentUrl;
const cardId = btn.dataset.cardId;
const panel = document.getElementById(cardId + '-requests');
if (!panel) return;

if (panel.style.display !== 'none') {
panel.style.display = 'none';
return;
}

panel.style.display = 'block';
panel.innerHTML = '<div style="padding: var(--space-3); color: var(--text-secondary);">Loading requests...</div>';

try {
const res = await fetch(`/api/registry/agents/${encodeURIComponent(agentUrl)}/monitoring/requests?limit=50`, {
credentials: 'include',
});
if (!res.ok) throw new Error('Failed to load');
const data = await res.json();

if (data.requests.length === 0) {
panel.innerHTML = '<div style="padding: var(--space-3); color: var(--text-secondary);">No outbound requests recorded yet.</div>';
return;
}

const rows = data.requests.map(function(r) {
const time = new Date(r.created_at).toLocaleString();
const status = r.success
? '<span style="color: var(--green-600);">OK</span>'
: '<span style="color: var(--red-600);">ERR</span>';
const duration = r.response_time_ms != null ? r.response_time_ms + 'ms' : '-';
const ua = r.user_agent ? r.user_agent.split(' ')[0] : '-';
return '<tr>'
+ '<td style="padding: 4px 8px;">' + escapeHtml(time) + '</td>'
+ '<td style="padding: 4px 8px;">' + escapeHtml(r.request_type) + '</td>'
+ '<td style="padding: 4px 8px;" title="' + escapeHtml(r.user_agent || '') + '">' + escapeHtml(ua) + '</td>'
+ '<td style="padding: 4px 8px;">' + status + '</td>'
+ '<td style="padding: 4px 8px;">' + escapeHtml(duration) + '</td>'
+ (r.error_message ? '<td style="padding: 4px 8px; color: var(--text-secondary); font-size: var(--text-xs);">' + escapeHtml(r.error_message) + '</td>' : '<td></td>')
+ '</tr>';
}).join('');

panel.innerHTML = '<div style="padding: var(--space-3);"><div style="font-weight: 600; margin-bottom: var(--space-2);">Outbound Requests (' + escapeHtml(String(data.total)) + ' total)</div>'
+ '<table style="width: 100%; font-size: var(--text-sm); border-collapse: collapse;">'
+ '<thead><tr style="text-align: left; color: var(--text-secondary);"><th style="padding: 4px 8px;">Time</th><th style="padding: 4px 8px;">Type</th><th style="padding: 4px 8px;">User-Agent</th><th style="padding: 4px 8px;">Status</th><th style="padding: 4px 8px;">Duration</th><th style="padding: 4px 8px;">Error</th></tr></thead>'
+ '<tbody>' + rows + '</tbody></table></div>';
} catch {
panel.innerHTML = '<div style="padding: var(--space-3); color: var(--red-600);">Failed to load requests.</div>';
}
});

// Track pill click — fetch history and show scenario details for that track
document.addEventListener('click', async function(e) {
const trackBtn = e.target.closest('.agent-track[data-track]');
Expand Down
7 changes: 4 additions & 3 deletions server/src/adagents-manager.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import axios from 'axios';
import { PropertyDefinition, PlacementDefinition } from './types.js';
import { AAO_UA_VALIDATOR } from './config/user-agents.js';

export interface ValidationError {
field: string;
Expand Down Expand Up @@ -175,7 +176,7 @@ export class AdAgentsManager {
timeout: 10000,
headers: {
'Accept': 'application/json',
'User-Agent': 'AdCP-Testing-Framework/1.0'
'User-Agent': AAO_UA_VALIDATOR
},
validateStatus: () => true, // Don't throw on non-2xx status codes
responseType: 'arraybuffer',
Expand Down Expand Up @@ -314,7 +315,7 @@ export class AdAgentsManager {
timeout: 10000,
headers: {
'Accept': 'application/json',
'User-Agent': 'AdCP-Testing-Framework/1.0'
'User-Agent': AAO_UA_VALIDATOR
},
validateStatus: () => true,
responseType: 'arraybuffer',
Expand Down Expand Up @@ -1229,7 +1230,7 @@ export class AdAgentsManager {
timeout: 3000, // Keep short for responsive UX
headers: {
'Accept': 'application/json',
'User-Agent': 'AdCP-Testing-Framework/1.0'
'User-Agent': AAO_UA_VALIDATOR
},
validateStatus: () => true
});
Expand Down
21 changes: 21 additions & 0 deletions server/src/addie/jobs/compliance-heartbeat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import { query } from '../../db/client.js';
import { notifyComplianceChange } from '../../notifications/compliance.js';
import { notifySystemError } from '../error-notifier.js';
import { logger as baseLogger } from '../../logger.js';
import { logOutboundRequest } from '../../db/outbound-log-db.js';
import { AAO_UA_COMPLIANCE } from '../../config/user-agents.js';

const logger = baseLogger.child({ module: 'compliance-heartbeat' });
const complianceDb = new ComplianceDatabase();
Expand Down Expand Up @@ -48,6 +50,7 @@ export async function runComplianceHeartbeatJob(options: HeartbeatOptions = {}):
);

for (const agent of agentsDue) {
const startTime = Date.now();
try {
// Use the owning org's saved credentials from agent_contexts.
// These are credentials the owner saved when connecting through Addie.
Expand All @@ -58,10 +61,19 @@ export async function runComplianceHeartbeatJob(options: HeartbeatOptions = {}):
dry_run: true,
timeout_ms: 60_000,
auth,
userAgent: AAO_UA_COMPLIANCE,
};

const complianceResult = await comply(agent.agent_url, complyOptions);

logOutboundRequest({
agent_url: agent.agent_url,
request_type: 'compliance',
user_agent: AAO_UA_COMPLIANCE,
response_time_ms: Date.now() - startTime,
success: true,
});

// Map track results to storage format
const tracksJson: TrackSummaryEntry[] = complianceResult.tracks.map(t => ({
track: t.track,
Expand Down Expand Up @@ -127,6 +139,15 @@ export async function runComplianceHeartbeatJob(options: HeartbeatOptions = {}):
} catch (error) {
logger.error({ error, agentUrl: agent.agent_url }, 'Compliance check failed for agent');

logOutboundRequest({
agent_url: agent.agent_url,
request_type: 'compliance',
user_agent: AAO_UA_COMPLIANCE,
response_time_ms: Date.now() - startTime,
success: false,
error_message: error instanceof Error ? error.message : 'Unknown error',
});

// Record failure so stale passing data doesn't persist
try {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
Expand Down
15 changes: 15 additions & 0 deletions server/src/addie/jobs/job-definitions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,20 @@ export function registerAllJobs(): void {
shouldLogResult: (r) => r.checked > 0,
});

// Outbound request log cleanup - retain 30 days
jobScheduler.register({
name: 'outbound-log-cleanup',
description: 'Clean up old outbound request logs',
interval: { value: 24, unit: 'hours' },
initialDelay: { value: 60, unit: 'minutes' },
runner: async () => {
const { cleanupOldRequests } = await import('../../db/outbound-log-db.js');
const deleted = await cleanupOldRequests(30);
return { deleted };
},
shouldLogResult: (r: { deleted: number }) => r.deleted > 0,
});

// Shadow evaluator - generates what Addie would have said and compares with human answers
jobScheduler.register({
name: 'shadow-evaluator',
Expand Down Expand Up @@ -536,4 +550,5 @@ export const JOB_NAMES = {
SHADOW_EVALUATOR: 'shadow-evaluator',
KNOWLEDGE_GAP_CLOSER: 'knowledge-gap-closer',
BRAND_REGISTRY_SWEEP: 'brand-registry-sweep',
OUTBOUND_LOG_CLEANUP: 'outbound-log-cleanup',
} as const;
3 changes: 2 additions & 1 deletion server/src/brand-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type {
ResolvedBrand,
KellerType,
} from './types';
import { AAO_UA_VALIDATOR } from './config/user-agents.js';

export interface BrandValidationError {
field: string;
Expand Down Expand Up @@ -168,7 +169,7 @@ export class BrandManager {
timeout: 10000,
headers: {
Accept: 'application/json',
'User-Agent': 'AdCP-Brand-Validator/1.0',
'User-Agent': AAO_UA_VALIDATOR,
},
validateStatus: () => true,
responseType: 'arraybuffer',
Expand Down
24 changes: 22 additions & 2 deletions server/src/capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import type { Agent } from "./types.js";
import { FormatsService } from "./formats.js";
import { createLogger } from "./logger.js";
import { is401Error, AuthenticationRequiredError } from "@adcp/client";
import { AAO_UA_DISCOVERY } from "./config/user-agents.js";
import { logOutboundRequest } from "./db/outbound-log-db.js";

const logger = createLogger('capabilities');

Expand Down Expand Up @@ -66,10 +68,19 @@ export class CapabilityDiscovery {
return cached;
}

const startTime = Date.now();
try {
const protocol = agent.protocol || "mcp";
const tools = await this.discoverTools(agent.url, protocol);

logOutboundRequest({
agent_url: agent.url,
request_type: 'discovery',
user_agent: AAO_UA_DISCOVERY,
response_time_ms: Date.now() - startTime,
success: true,
});

const profile: AgentCapabilityProfile = {
agent_url: agent.url,
protocol,
Expand All @@ -93,6 +104,15 @@ export class CapabilityDiscovery {
this.cache.set(agent.url, profile);
return profile;
} catch (error: any) {
logOutboundRequest({
agent_url: agent.url,
request_type: 'discovery',
user_agent: AAO_UA_DISCOVERY,
response_time_ms: Date.now() - startTime,
success: false,
error_message: error.message,
});

const isOAuthError = error instanceof AuthenticationRequiredError;
const errorProfile: AgentCapabilityProfile = {
agent_url: agent.url,
Expand Down Expand Up @@ -127,7 +147,7 @@ export class CapabilityDiscovery {
name: "Discovery Client",
agent_uri: url,
protocol: "mcp",
}]);
}], { userAgent: AAO_UA_DISCOVERY });
const client = multiClient.agent("discovery");

const agentInfo = await client.getAgentInfo();
Expand Down Expand Up @@ -164,7 +184,7 @@ export class CapabilityDiscovery {
name: "Discovery Client",
agent_uri: url,
protocol: "a2a",
}]);
}], { userAgent: AAO_UA_DISCOVERY });
const client = multiClient.agent("discovery");

const agentInfo = await client.getAgentInfo();
Expand Down
11 changes: 11 additions & 0 deletions server/src/config/user-agents.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* User-Agent strings for automated outbound requests to agent endpoints.
* Follows RFC 9110 convention with a +URL suffix pointing to documentation.
*/

const INFO_URL = 'https://agenticadvertising.org/docs/monitoring';

export const AAO_UA_HEALTH_CHECK = `AAO-HealthCheck/1.0 (+${INFO_URL})`;
export const AAO_UA_DISCOVERY = `AAO-Discovery/1.0 (+${INFO_URL})`;
export const AAO_UA_COMPLIANCE = `AAO-ComplianceCheck/1.0 (+${INFO_URL})`;
export const AAO_UA_VALIDATOR = `AAO-Validator/1.0 (+${INFO_URL})`;
Loading
Loading