diff --git a/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/DesignerCommandBarV2.tsx b/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/DesignerCommandBarV2.tsx index 0499be37aaf..80f23ea1a07 100644 --- a/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/DesignerCommandBarV2.tsx +++ b/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/DesignerCommandBarV2.tsx @@ -120,6 +120,7 @@ export const DesignerCommandBar = ({ isDesignerView, isMonitoringView, isCodeView, + isEvaluateView, isDarkMode, isUnitTest, isDraftMode, @@ -127,6 +128,7 @@ export const DesignerCommandBar = ({ showMonitoringView, showDesignerView, showCodeView, + showEvaluateView, switchWorkflowMode, }: { id: string; @@ -143,6 +145,7 @@ export const DesignerCommandBar = ({ isDesignerView?: boolean; isMonitoringView?: boolean; isCodeView?: boolean; + isEvaluateView?: boolean; isDarkMode: boolean; isUnitTest: boolean; isDraftMode?: boolean; @@ -151,6 +154,7 @@ export const DesignerCommandBar = ({ showMonitoringView: () => void; showDesignerView: () => void; showCodeView: () => void; + showEvaluateView: () => void; switchWorkflowMode: (draftMode: boolean) => void; }) => { const styles = useStyles(); @@ -348,6 +352,18 @@ export const DesignerCommandBar = ({ > Run history + ); diff --git a/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/laDesignerV2.tsx b/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/laDesignerV2.tsx index ee72ceebe32..1cbe6e55951 100644 --- a/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/laDesignerV2.tsx +++ b/apps/Standalone/src/designer/app/AzureLogicAppsDesigner/laDesignerV2.tsx @@ -46,6 +46,7 @@ import { StandardOperationManifestService, StandardRunService, StandardSearchService, + StandardEvaluationService, clone, equals, guid, @@ -75,6 +76,7 @@ import { setIsWorkflowDirty, setFocusNode, changePanelNode, + EvaluateView, } from '@microsoft/logic-apps-designer-v2'; import axios from 'axios'; import isEqual from 'lodash.isequal'; @@ -130,6 +132,7 @@ const DesignerEditor = () => { const [workflow, setWorkflow] = useState(); // Current workflow on the designer const [isDesignerView, setIsDesignerView] = useState(true); const [isCodeView, setIsCodeView] = useState(false); + const [isEvaluateView, setIsEvaluateView] = useState(false); const [isDraftMode, setIsDraftMode] = useState(true); const codeEditorRef = useRef<{ getValue: () => string | undefined; hasChanges: () => boolean }>(null); @@ -284,6 +287,7 @@ const DesignerEditor = () => { if (!isMonitoringView) { setIsDesignerView(false); setIsCodeView(false); + setIsEvaluateView(false); toggleMonitoringView(); } }, [isMonitoringView, toggleMonitoringView]); @@ -535,6 +539,12 @@ const DesignerEditor = () => { return; } + if (isEvaluateView) { + setIsEvaluateView(false); + setIsCodeView(true); + return; + } + if (isMonitoringView) { hideMonitoringView(); setIsCodeView(true); @@ -551,6 +561,12 @@ const DesignerEditor = () => { return; } + if (isEvaluateView) { + setIsEvaluateView(false); + setIsDesignerView(true); + return; + } + if (isMonitoringView) { hideMonitoringView(); setIsDesignerView(true); @@ -579,6 +595,18 @@ const DesignerEditor = () => { } }; + const showEvaluateView = useCallback(() => { + if (isEvaluateView) { + return; + } + if (isMonitoringView) { + hideMonitoringView(); + } + setIsDesignerView(false); + setIsCodeView(false); + setIsEvaluateView(true); + }, [isEvaluateView, isMonitoringView, hideMonitoringView]); + // Our iframe root element is given a strange padding (not in this repo), this removes it useEffect(() => { const root = document.getElementById('root'); @@ -685,11 +713,13 @@ const DesignerEditor = () => { isMonitoringView={isMonitoringView} isDesignerView={isDesignerView} isCodeView={isCodeView} + isEvaluateView={isEvaluateView} enableCopilot={() => dispatch(setIsChatBotEnabled(!showChatBot))} saveWorkflowFromCode={saveWorkflowFromCode} showMonitoringView={showMonitoringView} showDesignerView={showDesignerView} showCodeView={showCodeView} + showEvaluateView={showEvaluateView} switchWorkflowMode={switchWorkflowMode} isDraftMode={isDraftMode} prodWorkflow={artifactData?.properties.files[Artifact.WorkflowFile]} @@ -736,7 +766,7 @@ const DesignerEditor = () => { }} /> - {!isCodeView && ( + {!isCodeView && !isEvaluateView && (
{
)} {isCodeView && } + {isEvaluateView && } @@ -1102,6 +1133,10 @@ const getDesignerServices = ( httpClient, }); + const evaluationService = new StandardEvaluationService({ + ...defaultServiceParams, + }); + const roleService = new BaseRoleService({ baseUrl: armUrl, httpClient, @@ -1175,6 +1210,7 @@ const getDesignerServices = ( cognitiveServiceService, connectionParameterEditorService, editorService, + evaluationService, userPreferenceService: new BaseUserPreferenceService(), experimentationService: new BaseExperimentationService(), }; diff --git a/apps/vs-code-react/src/app/designer/DesignerCommandBar/indexV2.tsx b/apps/vs-code-react/src/app/designer/DesignerCommandBar/indexV2.tsx index 6598e226486..f4b6118b465 100644 --- a/apps/vs-code-react/src/app/designer/DesignerCommandBar/indexV2.tsx +++ b/apps/vs-code-react/src/app/designer/DesignerCommandBar/indexV2.tsx @@ -88,9 +88,11 @@ export interface DesignerCommandBarProps { isDesignerView: boolean; isCodeView: boolean; isMonitoringView: boolean; + isEvaluateView: boolean; switchToDesignerView: () => void; switchToCodeView: () => void; switchToMonitoringView: () => void; + switchToEvaluateView: () => void; } export const DesignerCommandBar: React.FC = ({ @@ -104,9 +106,11 @@ export const DesignerCommandBar: React.FC = ({ isDesignerView, isCodeView, isMonitoringView, + isEvaluateView, switchToDesignerView, switchToCodeView, switchToMonitoringView, + switchToEvaluateView, }) => { const vscode = useContext(VSCodeContext); const dispatch = DesignerStore.dispatch; @@ -275,6 +279,18 @@ export const DesignerCommandBar: React.FC = ({ > Run history + ); diff --git a/apps/vs-code-react/src/app/designer/appV2.tsx b/apps/vs-code-react/src/app/designer/appV2.tsx index 586056d2b55..c72d638af48 100644 --- a/apps/vs-code-react/src/app/designer/appV2.tsx +++ b/apps/vs-code-react/src/app/designer/appV2.tsx @@ -15,6 +15,7 @@ import { useThemeObserver, FloatingRunButton, useRun, + EvaluateView, } from '@microsoft/logic-apps-designer-v2'; import { BundleVersionRequirements, guid, isNullOrUndefined, isVersionSupported, Theme } from '@microsoft/logic-apps-shared'; import type { FileSystemConnectionInfo, MessageToVsix, StandardApp } from '@microsoft/vscode-extension-logic-apps'; @@ -54,6 +55,7 @@ export const DesignerApp = () => { const isDesignerView = useMemo(() => currentView === DesignerViewType.Workflow, [currentView]); const isCodeView = useMemo(() => currentView === DesignerViewType.Code, [currentView]); const isMonitoringView = useMemo(() => currentView === DesignerViewType.Monitoring, [currentView]); + const isEvaluateView = useMemo(() => currentView === DesignerViewType.Evaluate, [currentView]); const [runId, setRunId] = useState(_runId); @@ -267,35 +269,48 @@ export const DesignerApp = () => { hideMonitoringView(); setCurrentView(DesignerViewType.Workflow); } - }, [isDesignerView, isCodeView, isMonitoringView, validateAndSaveCodeView, hideMonitoringView]); + if (isEvaluateView) { + setCurrentView(DesignerViewType.Workflow); + } + }, [isDesignerView, isCodeView, isMonitoringView, isEvaluateView, validateAndSaveCodeView, hideMonitoringView]); const switchToCodeView = useCallback(async () => { if (isCodeView) { return; } - if (isDesignerView) { + if (isDesignerView || isEvaluateView) { setCurrentView(DesignerViewType.Code); } if (isMonitoringView) { hideMonitoringView(); setCurrentView(DesignerViewType.Code); } - }, [hideMonitoringView, isCodeView, isDesignerView, isMonitoringView]); + }, [hideMonitoringView, isCodeView, isDesignerView, isMonitoringView, isEvaluateView]); const switchToMonitoringView = useCallback(async () => { if (isMonitoringView) { return; } - if (isDesignerView) { + if (isDesignerView || isEvaluateView) { setCurrentView(DesignerViewType.Monitoring); } if (isCodeView) { validateAndSaveCodeView().then(() => setCurrentView(DesignerViewType.Monitoring)); } - }, [isMonitoringView, isDesignerView, isCodeView, validateAndSaveCodeView]); + }, [isMonitoringView, isDesignerView, isCodeView, isEvaluateView, validateAndSaveCodeView]); + + const switchToEvaluateView = useCallback(async () => { + if (isEvaluateView) { + return; + } + if (isMonitoringView) { + hideMonitoringView(); + } + setCurrentView(DesignerViewType.Evaluate); + }, [isEvaluateView, isMonitoringView, hideMonitoringView]); ///////////////////////////////////////////////////////////////////////////// // Rendering @@ -352,12 +367,14 @@ export const DesignerApp = () => { isDesignerView={isDesignerView} isCodeView={isCodeView} isMonitoringView={isMonitoringView} + isEvaluateView={isEvaluateView} switchToDesignerView={switchToDesignerView} switchToCodeView={switchToCodeView} switchToMonitoringView={switchToMonitoringView} + switchToEvaluateView={switchToEvaluateView} /> - {!isCodeView && ( + {!isCodeView && !isEvaluateView && (
{
)} {isCodeView && } + {isEvaluateView && } ) : null} diff --git a/apps/vs-code-react/src/app/designer/constants.ts b/apps/vs-code-react/src/app/designer/constants.ts index 7b252884ccf..898cadba176 100644 --- a/apps/vs-code-react/src/app/designer/constants.ts +++ b/apps/vs-code-react/src/app/designer/constants.ts @@ -67,4 +67,5 @@ export const DesignerViewType = { Workflow: 'workflow', Code: 'code', Monitoring: 'monitoring', + Evaluate: 'evaluate', }; diff --git a/apps/vs-code-react/src/app/designer/servicesHelper.ts b/apps/vs-code-react/src/app/designer/servicesHelper.ts index 4e08c05e49a..a8ce7457e70 100644 --- a/apps/vs-code-react/src/app/designer/servicesHelper.ts +++ b/apps/vs-code-react/src/app/designer/servicesHelper.ts @@ -6,6 +6,7 @@ import { StandardSearchService, BaseGatewayService, StandardRunService, + StandardEvaluationService, StandardArtifactService, BaseApiManagementService, BaseFunctionService, @@ -60,6 +61,7 @@ export interface IDesignerServices { loggerService: ILoggerService; connectionParameterEditorService: CustomConnectionParameterEditorService; cognitiveServiceService: BaseCognitiveServiceService; + evaluationService: StandardEvaluationService; } export const getDesignerServices = ( @@ -378,6 +380,12 @@ export const getDesignerServices = ( httpClient, }); + const evaluationService = new StandardEvaluationService({ + apiVersion, + baseUrl: isEmptyString(workflowRuntimeBaseUrl) ? baseUrl : workflowRuntimeBaseUrl, + httpClient, + }); + // MSI is not supported in VS Code const roleService = new BaseRoleService({ baseUrl: armUrl, @@ -428,6 +436,7 @@ export const getDesignerServices = ( loggerService, connectionParameterEditorService, cognitiveServiceService, + evaluationService, functionService, }; }; diff --git a/libs/designer-v2/src/lib/core/index.ts b/libs/designer-v2/src/lib/core/index.ts index c907fb87d64..1a08c3293cb 100644 --- a/libs/designer-v2/src/lib/core/index.ts +++ b/libs/designer-v2/src/lib/core/index.ts @@ -129,3 +129,10 @@ export { setLocation, setSubscription, setResourceGroup } from './state/template export { getConsumptionWorkflowPayloadForCreate } from './templates/utils/createhelper'; export * from './state/modal/modalSelectors'; export * from './state/modal/modalSlice'; +export * from './queries/evaluations'; +export { + setSelectedRun as setEvaluationSelectedRun, + setSelectedAction as setEvaluationSelectedAction, + resetEvaluationState, +} from './state/evaluation/evaluationSlice'; +export type { AgentAction, WorkflowRunEntry } from './state/evaluation/evaluationInterfaces'; diff --git a/libs/designer-v2/src/lib/core/queries/evaluations.ts b/libs/designer-v2/src/lib/core/queries/evaluations.ts new file mode 100644 index 00000000000..8c34d152396 --- /dev/null +++ b/libs/designer-v2/src/lib/core/queries/evaluations.ts @@ -0,0 +1,267 @@ +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; +import { EvaluationService } from '@microsoft/logic-apps-shared'; +import type { Evaluator } from '@microsoft/logic-apps-shared'; +import { useIsAgenticWorkflow } from '../state/designerView/designerViewSelectors'; + +const queryOpts = { + cacheTime: 1000 * 60 * 5, + refetchOnMount: true, + refetchOnWindowFocus: false, + refetchOnReconnect: false, +}; + +export const evaluationQueryKeys = { + evaluators: 'evaluators', + evaluator: 'evaluator', + evaluations: 'evaluations', + evaluation: 'evaluation', +}; + +export const useEvaluators = (workflowName: string, agentActionName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const globalAgentEvaluators = useGlobalAgentEvaluators(workflowName, !isAgenticWorkflow); + const agentEvaluators = useAgentEvaluators(workflowName, agentActionName, isAgenticWorkflow); + return isAgenticWorkflow ? agentEvaluators : globalAgentEvaluators; +}; + +const useGlobalAgentEvaluators = (workflowName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluators, workflowName], + async () => { + return EvaluationService().getGlobalAgentEvaluators(workflowName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim(), + } + ); +}; + +const useAgentEvaluators = (workflowName: string, agentActionName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluators, workflowName, agentActionName], + async () => { + return EvaluationService().getAgentEvaluators(workflowName, agentActionName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!agentActionName.trim(), + } + ); +}; + +export const useEvaluator = (workflowName: string, agentActionName: string, evaluatorName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const globalAgentEvaluator = useGlobalAgentEvaluator(workflowName, evaluatorName, !isAgenticWorkflow); + const agentEvaluator = useAgentEvaluator(workflowName, agentActionName, evaluatorName, isAgenticWorkflow); + return isAgenticWorkflow ? agentEvaluator : globalAgentEvaluator; +}; + +const useGlobalAgentEvaluator = (workflowName: string, evaluatorName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluator, workflowName, evaluatorName], + async () => { + return EvaluationService().getGlobalAgentEvaluator(workflowName, evaluatorName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!evaluatorName.trim(), + } + ); +}; + +const useAgentEvaluator = (workflowName: string, agentActionName: string, evaluatorName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluator, workflowName, agentActionName, evaluatorName], + async () => { + return EvaluationService().getAgentEvaluator(workflowName, agentActionName, evaluatorName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!agentActionName.trim() && !!evaluatorName.trim(), + } + ); +}; + +export const useCreateOrUpdateEvaluator = (workflowName: string, agentActionName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const createOrUpdateGlobalAgentEvaluator = useCreateOrUpdateGlobalAgentEvaluator(workflowName); + const createOrUpdateAgentEvaluator = useCreateOrUpdateAgentEvaluator(workflowName, agentActionName); + return isAgenticWorkflow ? createOrUpdateAgentEvaluator : createOrUpdateGlobalAgentEvaluator; +}; + +const useCreateOrUpdateGlobalAgentEvaluator = (workflowName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async ({ evaluatorName, evaluator }: { evaluatorName: string; evaluator: Evaluator }) => { + return EvaluationService().createOrUpdateGlobalAgentEvaluator(workflowName, evaluatorName, evaluator); + }, + { + onSuccess: (data, variables) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluators, workflowName]); + queryClient.invalidateQueries([evaluationQueryKeys.evaluator, workflowName, variables.evaluatorName]); + }, + } + ); +}; + +const useCreateOrUpdateAgentEvaluator = (workflowName: string, agentActionName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async ({ evaluatorName, evaluator }: { evaluatorName: string; evaluator: Evaluator }) => { + return EvaluationService().createOrUpdateAgentEvaluator(workflowName, agentActionName, evaluatorName, evaluator); + }, + { + onSuccess: (data, variables) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluators, workflowName, agentActionName]); + queryClient.invalidateQueries([evaluationQueryKeys.evaluator, workflowName, agentActionName, variables.evaluatorName]); + }, + } + ); +}; + +export const useDeleteEvaluator = (workflowName: string, agentActionName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const deleteGlobalAgentEvaluator = useDeleteGlobalAgentEvaluator(workflowName); + const deleteAgentEvaluator = useDeleteAgentEvaluator(workflowName, agentActionName); + return isAgenticWorkflow ? deleteAgentEvaluator : deleteGlobalAgentEvaluator; +}; + +const useDeleteGlobalAgentEvaluator = (workflowName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async (evaluatorName: string) => { + return EvaluationService().deleteGlobalAgentEvaluator(workflowName, evaluatorName); + }, + { + onSuccess: (data, evaluatorName) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluators, workflowName]); + queryClient.invalidateQueries([evaluationQueryKeys.evaluator, workflowName, evaluatorName]); + }, + } + ); +}; + +const useDeleteAgentEvaluator = (workflowName: string, agentActionName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async (evaluatorName: string) => { + return EvaluationService().deleteAgentEvaluator(workflowName, agentActionName, evaluatorName); + }, + { + onSuccess: (data, evaluatorName) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluators, workflowName, agentActionName]); + queryClient.invalidateQueries([evaluationQueryKeys.evaluator, workflowName, agentActionName, evaluatorName]); + }, + } + ); +}; + +export const useEvaluations = (workflowName: string, runId: string, agentActionName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const globalAgentEvaluations = useGlobalAgentEvaluations(workflowName, runId, !isAgenticWorkflow); + const agentEvaluations = useAgentEvaluations(workflowName, runId, agentActionName, isAgenticWorkflow); + return isAgenticWorkflow ? agentEvaluations : globalAgentEvaluations; +}; + +const useGlobalAgentEvaluations = (workflowName: string, runId: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluations, workflowName, runId], + async () => { + return EvaluationService().getGlobalAgentEvaluations(workflowName, runId); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!runId.trim(), + } + ); +}; + +const useAgentEvaluations = (workflowName: string, runId: string, agentActionName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluations, workflowName, runId, agentActionName], + async () => { + return EvaluationService().getAgentEvaluations(workflowName, runId, agentActionName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!runId.trim() && !!agentActionName.trim(), + } + ); +}; + +export const useEvaluation = (workflowName: string, runId: string, agentActionName: string, evaluatorName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const globalAgentEvaluation = useGlobalAgentEvaluation(workflowName, runId, evaluatorName, !isAgenticWorkflow); + const agentEvaluation = useAgentEvaluation(workflowName, runId, agentActionName, evaluatorName, isAgenticWorkflow); + return isAgenticWorkflow ? agentEvaluation : globalAgentEvaluation; +}; + +const useGlobalAgentEvaluation = (workflowName: string, runId: string, evaluatorName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluation, workflowName, runId, evaluatorName], + async () => { + return EvaluationService().getGlobalAgentEvaluation(workflowName, runId, evaluatorName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!runId.trim() && !!evaluatorName.trim(), + } + ); +}; + +const useAgentEvaluation = (workflowName: string, runId: string, agentActionName: string, evaluatorName: string, isEnabled: boolean) => { + return useQuery( + [evaluationQueryKeys.evaluation, workflowName, runId, agentActionName, evaluatorName], + async () => { + return EvaluationService().getAgentEvaluation(workflowName, runId, agentActionName, evaluatorName); + }, + { + ...queryOpts, + enabled: isEnabled && !!workflowName.trim() && !!runId.trim() && !!agentActionName.trim() && !!evaluatorName.trim(), + } + ); +}; + +export const useRunEvaluation = (workflowName: string, agentActionName: string) => { + const isAgenticWorkflow = useIsAgenticWorkflow(); + const runGlobalAgentEvaluation = useRunGlobalAgentEvaluation(workflowName); + const runAgentEvaluation = useRunAgentEvaluation(workflowName, agentActionName); + return isAgenticWorkflow ? runAgentEvaluation : runGlobalAgentEvaluation; +}; + +const useRunGlobalAgentEvaluation = (workflowName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async ({ runId, evaluatorName }: { runId: string; evaluatorName: string }) => { + return EvaluationService().runGlobalAgentEvaluation(workflowName, runId, evaluatorName); + }, + { + onSuccess: (data, variables) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluations, workflowName, variables.runId]); + queryClient.invalidateQueries([evaluationQueryKeys.evaluation, workflowName, variables.runId, variables.evaluatorName]); + }, + } + ); +}; + +const useRunAgentEvaluation = (workflowName: string, agentActionName: string) => { + const queryClient = useQueryClient(); + return useMutation( + async ({ runId, evaluatorName }: { runId: string; evaluatorName: string }) => { + return EvaluationService().runAgentEvaluation(workflowName, runId, agentActionName, evaluatorName); + }, + { + onSuccess: (data, variables) => { + queryClient.invalidateQueries([evaluationQueryKeys.evaluations, workflowName, variables.runId, agentActionName]); + queryClient.invalidateQueries([ + evaluationQueryKeys.evaluation, + workflowName, + variables.runId, + agentActionName, + variables.evaluatorName, + ]); + }, + } + ); +}; diff --git a/libs/designer-v2/src/lib/core/queries/runs.ts b/libs/designer-v2/src/lib/core/queries/runs.ts index 835883b6dba..b1ec7b66bf2 100644 --- a/libs/designer-v2/src/lib/core/queries/runs.ts +++ b/libs/designer-v2/src/lib/core/queries/runs.ts @@ -415,10 +415,10 @@ export const useRunChatHistory = (runId: string | undefined, isEnabled: boolean) ); }; -export const useChatHistory = (isMonitoringView: boolean, runId: string | undefined, nodeIds: string[] = [], isA2AWorkflow: boolean) => { - const actionHistoryQuery = useActionsChatHistory(nodeIds, runId, isMonitoringView && !isA2AWorkflow); +export const useChatHistory = (isEnabled: boolean, runId: string | undefined, nodeIds: string[] = [], isA2AWorkflow: boolean) => { + const actionHistoryQuery = useActionsChatHistory(nodeIds, runId, isEnabled && !isA2AWorkflow); - const runHistoryQuery = useRunChatHistory(runId, isMonitoringView && isA2AWorkflow); + const runHistoryQuery = useRunChatHistory(runId, isEnabled && isA2AWorkflow); return isA2AWorkflow ? runHistoryQuery : actionHistoryQuery; }; diff --git a/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsInterfaces.ts b/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsInterfaces.ts index b2a7500c2fd..09d743f7417 100644 --- a/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsInterfaces.ts +++ b/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsInterfaces.ts @@ -24,6 +24,7 @@ import type { IUserPreferenceService, IExperimentationService, ICognitiveServiceService, + IEvaluationService, } from '@microsoft/logic-apps-shared'; import type { MaximumWaitingRunsMetadata } from '../../../ui/settings'; @@ -82,4 +83,5 @@ export interface ServiceOptions { userPreferenceService?: IUserPreferenceService; experimentationService?: IExperimentationService; cognitiveServiceService?: ICognitiveServiceService; + evaluationService?: IEvaluationService; } diff --git a/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsSlice.ts b/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsSlice.ts index ad8e7ccb4a2..cac27d07399 100644 --- a/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsSlice.ts +++ b/libs/designer-v2/src/lib/core/state/designerOptions/designerOptionsSlice.ts @@ -25,6 +25,7 @@ import { InitUserPreferenceService, InitExperimentationServiceService, InitCognitiveServiceService, + InitEvaluationService, } from '@microsoft/logic-apps-shared'; import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; import type { PayloadAction } from '@reduxjs/toolkit'; @@ -78,6 +79,7 @@ export const initializeServices = createAsyncThunk( userPreferenceService, experimentationService, cognitiveServiceService, + evaluationService, }: ServiceOptions) => { const loggerServices: ILoggerService[] = []; if (loggerService) { @@ -142,6 +144,10 @@ export const initializeServices = createAsyncThunk( InitCognitiveServiceService(cognitiveServiceService); } + if (evaluationService) { + InitEvaluationService(evaluationService); + } + // Experimentation service is being used to A/B test features in the designer so in case client does not want to use the A/B test feature, // we are always defaulting to the false implementation of the experimentation service. InitExperimentationServiceService(experimentationService); diff --git a/libs/designer-v2/src/lib/core/state/evaluation/evaluationInterfaces.ts b/libs/designer-v2/src/lib/core/state/evaluation/evaluationInterfaces.ts new file mode 100644 index 00000000000..eb78bff516d --- /dev/null +++ b/libs/designer-v2/src/lib/core/state/evaluation/evaluationInterfaces.ts @@ -0,0 +1,35 @@ +import type { Evaluator, EvaluationResult } from '@microsoft/logic-apps-shared'; + +export type RightPanelView = 'empty' | 'create' | 'edit' | 'view' | 'result'; + +export interface AgentAction { + name: string; + status: string; + startTime: string; + endTime: string; +} + +export interface WorkflowRunEntry { + id: string; + name: string; + startTime: string; + endTime: string; + status: string; +} + +export interface EvaluationState { + evaluators: Evaluator[]; + evaluatorsLoading: boolean; + selectedEvaluator: Evaluator | null; + selectedRun: WorkflowRunEntry | null; + selectedAction: AgentAction | null; + agentActions: AgentAction[]; + agentActionsLoading: boolean; + rightPanelView: RightPanelView; + editingEvaluator: Evaluator | null; + evaluationResult: EvaluationResult | null; + evaluationLoading: boolean; + evaluationError: string | null; + runningEvaluatorName: string; + searchQuery: string; +} diff --git a/libs/designer-v2/src/lib/core/state/evaluation/evaluationSelectors.ts b/libs/designer-v2/src/lib/core/state/evaluation/evaluationSelectors.ts new file mode 100644 index 00000000000..172b565e567 --- /dev/null +++ b/libs/designer-v2/src/lib/core/state/evaluation/evaluationSelectors.ts @@ -0,0 +1,47 @@ +import type { RootState } from '../../store'; +import { useMemo } from 'react'; +import { useSelector } from 'react-redux'; +import { useIsAgenticWorkflow } from '../designerView/designerViewSelectors'; + +export const useEvaluators = () => useSelector((state: RootState) => state.evaluation.evaluators); +export const useEvaluatorsLoading = () => useSelector((state: RootState) => state.evaluation.evaluatorsLoading); +export const useSelectedEvaluator = () => useSelector((state: RootState) => state.evaluation.selectedEvaluator); +export const useSelectedRun = () => useSelector((state: RootState) => state.evaluation.selectedRun); +export const useSelectedAction = () => useSelector((state: RootState) => state.evaluation.selectedAction); +export const useAgentActions = () => useSelector((state: RootState) => state.evaluation.agentActions); +export const useAgentActionsLoading = () => useSelector((state: RootState) => state.evaluation.agentActionsLoading); +export const useRightPanelView = () => useSelector((state: RootState) => state.evaluation.rightPanelView); +export const useEditingEvaluator = () => useSelector((state: RootState) => state.evaluation.editingEvaluator); +export const useEvaluationResult = () => useSelector((state: RootState) => state.evaluation.evaluationResult); +export const useEvaluationLoading = () => useSelector((state: RootState) => state.evaluation.evaluationLoading); +export const useEvaluationError = () => useSelector((state: RootState) => state.evaluation.evaluationError); +export const useRunningEvaluatorName = () => useSelector((state: RootState) => state.evaluation.runningEvaluatorName); +export const useSearchQuery = () => useSelector((state: RootState) => state.evaluation.searchQuery); + +export const useFilteredEvaluators = () => { + const evaluators = useEvaluators(); + const searchQuery = useSearchQuery(); + return useMemo( + () => + evaluators.filter( + (ev) => ev.name.toLowerCase().includes(searchQuery.toLowerCase()) || ev.template.toLowerCase().includes(searchQuery.toLowerCase()) + ), + [evaluators, searchQuery] + ); +}; + +export const useCanRunEvaluation = () => { + const selectedRun = useSelectedRun(); + const selectedAction = useSelectedAction(); + const isAgenticWorkflow = useIsAgenticWorkflow(); + + return useMemo(() => { + if (!selectedRun) { + return false; + } + if (isAgenticWorkflow) { + return !!selectedAction; + } + return true; + }, [selectedRun, selectedAction, isAgenticWorkflow]); +}; diff --git a/libs/designer-v2/src/lib/core/state/evaluation/evaluationSlice.ts b/libs/designer-v2/src/lib/core/state/evaluation/evaluationSlice.ts new file mode 100644 index 00000000000..727c12fc072 --- /dev/null +++ b/libs/designer-v2/src/lib/core/state/evaluation/evaluationSlice.ts @@ -0,0 +1,129 @@ +import type { Evaluator, EvaluationResult } from '@microsoft/logic-apps-shared'; +import type { PayloadAction } from '@reduxjs/toolkit'; +import { createSlice } from '@reduxjs/toolkit'; +import { resetWorkflowState } from '../global'; +import type { EvaluationState, RightPanelView, WorkflowRunEntry, AgentAction } from './evaluationInterfaces'; + +export const initialEvaluationState: EvaluationState = { + evaluators: [], + evaluatorsLoading: false, + selectedEvaluator: null, + selectedRun: null, + selectedAction: null, + agentActions: [], + agentActionsLoading: false, + rightPanelView: 'empty', + editingEvaluator: null, + evaluationResult: null, + evaluationLoading: false, + evaluationError: null, + runningEvaluatorName: '', + searchQuery: '', +}; + +export const evaluationSlice = createSlice({ + name: 'evaluation', + initialState: initialEvaluationState, + reducers: { + setEvaluators: (state, action: PayloadAction) => { + state.evaluators = action.payload; + }, + setEvaluatorsLoading: (state, action: PayloadAction) => { + state.evaluatorsLoading = action.payload; + }, + setSelectedEvaluator: (state, action: PayloadAction) => { + state.selectedEvaluator = action.payload; + if (action.payload) { + state.rightPanelView = 'view'; + } else { + state.rightPanelView = 'empty'; + } + }, + setSelectedRun: (state, action: PayloadAction) => { + state.selectedRun = action.payload; + state.selectedAction = null; + }, + setSelectedAction: (state, action: PayloadAction) => { + state.selectedAction = action.payload; + }, + setAgentActions: (state, action: PayloadAction) => { + state.agentActions = action.payload; + }, + setAgentActionsLoading: (state, action: PayloadAction) => { + state.agentActionsLoading = action.payload; + }, + setRightPanelView: (state, action: PayloadAction) => { + state.rightPanelView = action.payload; + }, + setEditingEvaluator: (state, action: PayloadAction) => { + state.editingEvaluator = action.payload; + }, + setEvaluationResult: (state, action: PayloadAction) => { + state.evaluationResult = action.payload; + }, + setEvaluationLoading: (state, action: PayloadAction) => { + state.evaluationLoading = action.payload; + }, + setEvaluationError: (state, action: PayloadAction) => { + state.evaluationError = action.payload; + }, + setRunningEvaluatorName: (state, action: PayloadAction) => { + state.runningEvaluatorName = action.payload; + }, + setSearchQuery: (state, action: PayloadAction) => { + state.searchQuery = action.payload; + }, + startCreateEvaluator: (state) => { + state.selectedEvaluator = null; + state.editingEvaluator = null; + state.rightPanelView = 'create'; + }, + startEditEvaluator: (state, action: PayloadAction) => { + state.selectedEvaluator = action.payload; + state.editingEvaluator = action.payload; + state.rightPanelView = 'edit'; + }, + finishFormAction: (state) => { + state.rightPanelView = 'empty'; + state.editingEvaluator = null; + state.selectedEvaluator = null; + }, + cancelFormAction: (state) => { + if (state.selectedEvaluator && state.rightPanelView === 'edit') { + state.rightPanelView = 'view'; + } else { + state.rightPanelView = 'empty'; + state.selectedEvaluator = null; + } + state.editingEvaluator = null; + }, + resetEvaluationState: () => initialEvaluationState, + }, + extraReducers: (builder) => { + builder.addCase(resetWorkflowState, () => initialEvaluationState); + }, +}); + +export const { + setEvaluators, + setEvaluatorsLoading, + setSelectedEvaluator, + setSelectedRun, + setSelectedAction, + setAgentActions, + setAgentActionsLoading, + setRightPanelView, + setEditingEvaluator, + setEvaluationResult, + setEvaluationLoading, + setEvaluationError, + setRunningEvaluatorName, + setSearchQuery, + startCreateEvaluator, + startEditEvaluator, + finishFormAction, + cancelFormAction, + resetEvaluationState, +} = evaluationSlice.actions; + +export default evaluationSlice.reducer; diff --git a/libs/designer-v2/src/lib/core/store.ts b/libs/designer-v2/src/lib/core/store.ts index 9738ea030a6..1dc1c3153bf 100644 --- a/libs/designer-v2/src/lib/core/store.ts +++ b/libs/designer-v2/src/lib/core/store.ts @@ -14,6 +14,7 @@ import workflowReducer from './state/workflow/workflowSlice'; import workflowParametersReducer from './state/workflowparameters/workflowparametersSlice'; import modalReducer from './state/modal/modalSlice'; import notesReducer from './state/notes/notesSlice'; +import evaluationReducer from './state/evaluation/evaluationSlice'; import { configureStore } from '@reduxjs/toolkit'; import type {} from 'redux-thunk'; @@ -42,6 +43,7 @@ export const store = configureStore({ undoRedo: undoRedoReducer, modal: modalReducer, notes: notesReducer, + evaluation: evaluationReducer, // if is in dev environment, add devSlice to store ...(process.env.NODE_ENV === 'development' ? { dev: devReducer } : {}), }, diff --git a/libs/designer-v2/src/lib/ui/evaluation/AgentChatPanel.tsx b/libs/designer-v2/src/lib/ui/evaluation/AgentChatPanel.tsx new file mode 100644 index 00000000000..379a89cef93 --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/AgentChatPanel.tsx @@ -0,0 +1,83 @@ +import type { ConversationItem } from '@microsoft/designer-ui'; +import { useEffect, useMemo, useState } from 'react'; +import { Spinner, Text } from '@fluentui/react-components'; +import { ChatbotUI } from '@microsoft/logic-apps-chatbot'; +import { useSelectedRun } from '../../core/state/evaluation/evaluationSelectors'; +import { useChatHistory } from '../../core/queries/runs'; +import { parseChatHistory } from '../panel/agentChat/helper'; +import { useIsA2AWorkflow } from '../../core/state/designerView/designerViewSelectors'; +import { useAgentOperations } from '../../core/state/workflow/workflowSelectors'; +import { useEvaluateViewStyles } from './EvaluateView.styles'; + +export const AgentChatPanel = () => { + const styles = useEvaluateViewStyles(); + const selectedRun = useSelectedRun(); + + const isA2AWorkflow = useIsA2AWorkflow(); + const agentOperations = useAgentOperations(); + const runId = selectedRun?.id; + + const { isFetching: isChatHistoryFetching, data: chatHistoryData } = useChatHistory(!!runId, runId, agentOperations, isA2AWorkflow); + + const [conversation, setConversation] = useState([]); + + const noopToolResult = useMemo(() => () => {}, []); + const noopToolContent = useMemo(() => () => {}, []); + const noopAgent = useMemo(() => () => {}, []); + + useEffect(() => { + if (chatHistoryData) { + const items = parseChatHistory(chatHistoryData, noopToolResult as any, noopToolContent as any, noopAgent, isA2AWorkflow); + setConversation(items); + } else { + setConversation([]); + } + }, [chatHistoryData, noopToolResult, noopToolContent, noopAgent, isA2AWorkflow]); + + if (!selectedRun) { + return ( +
+ Select a run to view chat history +
+ ); + } + + if (isChatHistoryFetching) { + return ( +
+ +
+ ); + } + + if (conversation.length === 0) { + return ( +
+ No chat history available for this run +
+ ); + } + + return ( +
+ Promise.resolve(), + readOnly: true, + }} + string={{ + submit: '', + progressState: '', + progressSave: '', + }} + body={{ + messages: conversation, + focus: false, + answerGenerationInProgress: false, + setFocus: () => {}, + }} + /> +
+ ); +}; diff --git a/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.styles.ts b/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.styles.ts new file mode 100644 index 00000000000..d7dbc35ea54 --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.styles.ts @@ -0,0 +1,372 @@ +import { makeStyles, tokens } from '@fluentui/react-components'; + +export const useEvaluateViewStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + height: '100%', + overflow: 'hidden', + }, + main: { + flex: '1', + display: 'flex', + overflow: 'hidden', + }, + panel: { + display: 'flex', + flexDirection: 'column', + overflow: 'hidden', + borderRight: `1px solid ${tokens.colorNeutralStroke1}`, + }, + panelRuns: { + width: '320px', + minWidth: '240px', + }, + panelEvaluators: { + width: '420px', + minWidth: '300px', + }, + panelDetail: { + flex: '1', + minWidth: '300px', + overflow: 'auto', + borderRight: 'none', + }, + panelHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + padding: '12px 16px', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + }, + panelHeaderWithBack: { + padding: '8px 16px', + }, + panelTitle: { + margin: '0', + fontSize: tokens.fontSizeBase400, + fontWeight: tokens.fontWeightSemibold, + }, + searchContainer: { + padding: '8px 16px', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + }, + listContainer: { + flex: '1', + overflow: 'auto', + }, + listItem: { + padding: '10px 16px', + cursor: 'pointer', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + '&:hover': { + backgroundColor: tokens.colorNeutralBackground1Hover, + }, + }, + listItemSelected: { + backgroundColor: tokens.colorNeutralBackground1Selected, + }, + runName: { + fontSize: tokens.fontSizeBase300, + fontWeight: tokens.fontWeightSemibold, + }, + runTiming: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + marginTop: '2px', + }, + emptyState: { + flex: '1', + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + color: tokens.colorNeutralForeground2, + padding: '32px', + textAlign: 'center', + }, + emptyTitle: { + fontSize: tokens.fontSizeBase300, + fontWeight: tokens.fontWeightSemibold, + margin: '0 0 4px 0', + }, + emptySubtext: { + fontSize: tokens.fontSizeBase200, + margin: '0', + opacity: 0.7, + }, + loadingContainer: { + display: 'flex', + justifyContent: 'center', + padding: '24px', + }, + // Table styles for evaluators panel + tableHeader: { + display: 'flex', + padding: '8px 16px', + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + fontWeight: tokens.fontWeightSemibold, + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + }, + tableRow: { + display: 'flex', + alignItems: 'center', + padding: '8px 16px', + cursor: 'pointer', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + '&:hover': { + backgroundColor: tokens.colorNeutralBackground1Hover, + }, + }, + tableRowSelected: { + backgroundColor: tokens.colorNeutralBackground1Selected, + }, + colType: { + width: '140px', + flexShrink: 0, + fontSize: tokens.fontSizeBase200, + }, + colName: { + flex: '1', + fontSize: tokens.fontSizeBase300, + fontWeight: tokens.fontWeightSemibold, + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + }, + colActions: { + display: 'flex', + gap: '4px', + flexShrink: 0, + }, + colResult: { + width: '70px', + flexShrink: 0, + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + }, + // Agent actions list + actionItem: { + display: 'flex', + alignItems: 'center', + gap: '12px', + padding: '10px 16px', + cursor: 'pointer', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + '&:hover': { + backgroundColor: tokens.colorNeutralBackground1Hover, + }, + }, + actionItemSelected: { + backgroundColor: tokens.colorNeutralBackground1Selected, + }, + actionIndex: { + width: '24px', + height: '24px', + borderRadius: '50%', + backgroundColor: tokens.colorNeutralBackground3, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + flexShrink: 0, + }, + actionDetails: { + flex: '1', + minWidth: 0, + }, + actionStatus: { + display: 'flex', + alignItems: 'center', + gap: '6px', + fontSize: tokens.fontSizeBase300, + }, + actionTiming: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + marginTop: '2px', + }, + // Form styles + formContent: { + flex: '1', + overflow: 'auto', + padding: '16px', + display: 'flex', + flexDirection: 'column', + gap: '16px', + }, + formActions: { + display: 'flex', + justifyContent: 'flex-end', + gap: '8px', + padding: '12px 16px', + borderTop: `1px solid ${tokens.colorNeutralStroke1}`, + }, + formError: { + padding: '8px 12px', + margin: '0 16px', + borderRadius: '4px', + backgroundColor: tokens.colorPaletteRedBackground1, + color: tokens.colorPaletteRedForeground1, + fontSize: tokens.fontSizeBase200, + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + }, + formRow: { + display: 'flex', + gap: '12px', + }, + formFieldHalf: { + flex: '1', + }, + // Tool calls + toolCallItem: { + padding: '12px', + borderRadius: '6px', + border: `1px solid ${tokens.colorNeutralStroke1}`, + display: 'flex', + flexDirection: 'column', + gap: '8px', + }, + toolCallHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + fontSize: tokens.fontSizeBase200, + fontWeight: tokens.fontWeightSemibold, + }, + // View panel + fieldRow: { + display: 'flex', + flexDirection: 'column', + gap: '4px', + padding: '8px 0', + }, + fieldLabel: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + fontWeight: tokens.fontWeightSemibold, + }, + fieldValue: { + fontSize: tokens.fontSizeBase300, + }, + promptValue: { + whiteSpace: 'pre-wrap', + backgroundColor: tokens.colorNeutralBackground3, + padding: '8px 12px', + borderRadius: '4px', + fontSize: tokens.fontSizeBase200, + }, + definitionActions: { + display: 'flex', + gap: '8px', + paddingTop: '12px', + }, + // Result panel + resultBadge: { + display: 'inline-flex', + padding: '4px 12px', + borderRadius: '12px', + fontWeight: tokens.fontWeightSemibold, + fontSize: tokens.fontSizeBase300, + }, + resultPassed: { + backgroundColor: tokens.colorPaletteGreenBackground1, + color: tokens.colorPaletteGreenForeground1, + }, + resultFailed: { + backgroundColor: tokens.colorPaletteRedBackground1, + color: tokens.colorPaletteRedForeground1, + }, + tokenStats: { + display: 'flex', + gap: '16px', + padding: '12px 0', + }, + tokenStat: { + display: 'flex', + flexDirection: 'column', + gap: '2px', + }, + statLabel: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + }, + statValue: { + fontSize: tokens.fontSizeBase400, + fontWeight: tokens.fontWeightSemibold, + }, + resultReason: { + padding: '12px', + backgroundColor: tokens.colorNeutralBackground3, + borderRadius: '6px', + fontSize: tokens.fontSizeBase200, + whiteSpace: 'pre-wrap', + }, + // Status indicators + statusSucceeded: { + color: tokens.colorPaletteGreenForeground1, + }, + statusFailed: { + color: tokens.colorPaletteRedForeground1, + }, + statusRunning: { + color: tokens.colorPaletteYellowForeground1, + }, + // Tabs + tabContainer: { + display: 'flex', + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + }, + // Card + card: { + padding: '16px', + borderRadius: '8px', + border: `1px solid ${tokens.colorNeutralStroke1}`, + backgroundColor: tokens.colorNeutralBackground1, + marginBottom: '16px', + }, + evaluationHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + marginBottom: '12px', + }, + resultSection: { + padding: '8px 0', + display: 'flex', + flexDirection: 'column', + gap: '8px', + }, + detailRow: { + display: 'flex', + justifyContent: 'space-between', + padding: '4px 0', + }, + detailLabel: { + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground2, + }, + detailValue: { + fontSize: tokens.fontSizeBase300, + fontWeight: tokens.fontWeightSemibold, + }, + panelChat: { + width: '380px', + minWidth: '280px', + borderLeft: `1px solid ${tokens.colorNeutralStroke1}`, + display: 'flex', + flexDirection: 'column', + overflow: 'hidden', + }, + chatPanelContainer: { + flex: '1', + overflow: 'auto', + display: 'flex', + flexDirection: 'column', + position: 'relative', + }, +}); diff --git a/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.tsx b/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.tsx new file mode 100644 index 00000000000..122f525ddea --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/EvaluateView.tsx @@ -0,0 +1,123 @@ +import { mergeClasses } from '@fluentui/react-components'; +import { useDispatch } from 'react-redux'; +import { + useSelectedEvaluator, + useRightPanelView, + useSelectedRun, + useSelectedAction, +} from '../../core/state/evaluation/evaluationSelectors'; +import { + startEditEvaluator, + setSelectedEvaluator, + setRightPanelView, + setEvaluationLoading, + setEvaluationError, + setEvaluationResult, + setRunningEvaluatorName, +} from '../../core/state/evaluation/evaluationSlice'; +import { useRunEvaluation, useDeleteEvaluator } from '../../core/queries/evaluations'; +import { RunDatasetPanel } from './RunDatasetPanel'; +import { EvaluatorManagementPanel } from './EvaluatorManagementPanel'; +import { EvaluatorFormPanel } from './EvaluatorFormPanel'; +import { EvaluatorDetailsPanel } from './EvaluatorDetailsPanel'; +import { EvaluationResultPanel } from './EvaluationResultPanel'; +import { AgentChatPanel } from './AgentChatPanel'; +import { useEvaluateViewStyles } from './EvaluateView.styles'; +import { useCallback } from 'react'; + +interface EvaluateViewProps { + workflowName: string; +} + +export const EvaluateView = ({ workflowName }: EvaluateViewProps) => { + const styles = useEvaluateViewStyles(); + const dispatch = useDispatch(); + const selectedEvaluator = useSelectedEvaluator(); + const rightPanelView = useRightPanelView(); + const selectedRun = useSelectedRun(); + const selectedAction = useSelectedAction(); + + const { mutateAsync: runEvaluation } = useRunEvaluation(workflowName, selectedAction?.name ?? ''); + const { mutateAsync: deleteEvaluator } = useDeleteEvaluator(workflowName, selectedAction?.name ?? ''); + + const handleRunClick = useCallback(async () => { + if (!selectedRun || !selectedEvaluator) { + return; + } + + dispatch(setRightPanelView('result')); + dispatch(setEvaluationLoading(true)); + dispatch(setEvaluationError(null)); + dispatch(setEvaluationResult(null)); + dispatch(setRunningEvaluatorName(selectedEvaluator.name)); + + try { + const result = await runEvaluation({ + runId: selectedRun.name, + evaluatorName: selectedEvaluator.name, + }); + dispatch(setEvaluationResult(result)); + } catch (err) { + dispatch(setEvaluationError(err instanceof Error ? err.message : 'Failed to run evaluation')); + } finally { + dispatch(setEvaluationLoading(false)); + } + }, [dispatch, selectedRun, selectedEvaluator, runEvaluation]); + + const handleDeleteClick = useCallback(async () => { + if (!selectedEvaluator) { + return; + } + try { + await deleteEvaluator(selectedEvaluator.name); + dispatch(setSelectedEvaluator(null)); + } catch (err) { + console.error('Failed to delete evaluator:', err); + } + }, [deleteEvaluator, dispatch, selectedEvaluator]); + + const renderRightPanel = () => { + switch (rightPanelView) { + case 'create': + case 'edit': + return ; + case 'view': + return selectedEvaluator ? ( + dispatch(startEditEvaluator(selectedEvaluator))} + onRun={handleRunClick} + onDelete={handleDeleteClick} + /> + ) : null; + case 'result': + return ; + default: + return ( +
+

Select an action to get started

+

Create a new evaluator or select one to view

+
+ ); + } + }; + + return ( +
+
+ + +
{renderRightPanel()}
+ {selectedRun && ( +
+
+

Chat History

+
+ +
+ )} +
+
+ ); +}; diff --git a/libs/designer-v2/src/lib/ui/evaluation/EvaluationResultPanel.tsx b/libs/designer-v2/src/lib/ui/evaluation/EvaluationResultPanel.tsx new file mode 100644 index 00000000000..e81f77c4b0f --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/EvaluationResultPanel.tsx @@ -0,0 +1,108 @@ +import { mergeClasses, Spinner } from '@fluentui/react-components'; +import { + useEvaluationResult, + useEvaluationLoading, + useEvaluationError, + useRunningEvaluatorName, +} from '../../core/state/evaluation/evaluationSelectors'; +import { useEvaluateViewStyles } from './EvaluateView.styles'; + +export const EvaluationResultPanel = () => { + const styles = useEvaluateViewStyles(); + const result = useEvaluationResult(); + const loading = useEvaluationLoading(); + const error = useEvaluationError(); + const evaluatorName = useRunningEvaluatorName(); + + if (loading) { + return ( +
+
+ +
+
+ ); + } + + if (error) { + return ( +
+
+
+

Evaluation Result

+

Evaluator: {evaluatorName}

+
+
+
+
+ {error} +
+
+
+ ); + } + + if (!result) { + return ( +
+
+

No evaluation result

+
+
+ ); + } + + const isPassed = result.result?.toLowerCase() === 'passed'; + + return ( +
+
+
+

Evaluation Result

+

Evaluator: {evaluatorName}

+
+
+ +
+
+
+ {result.result} + + Value: + {result.value} + +
+ + {result.agentActionName && ( +
+ Agent Action + {result.agentActionName} +
+ )} + + {result.reason && ( +
+ Reason +
{result.reason}
+
+ )} + +
+
+ Total Tokens + {result.totalTokens} +
+
+ Input Tokens + {result.inputTokens} +
+
+ Output Tokens + {result.outputTokens} +
+
+
+
+
+ ); +}; diff --git a/libs/designer-v2/src/lib/ui/evaluation/EvaluatorDetailsPanel.tsx b/libs/designer-v2/src/lib/ui/evaluation/EvaluatorDetailsPanel.tsx new file mode 100644 index 00000000000..e8d33809ec8 --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/EvaluatorDetailsPanel.tsx @@ -0,0 +1,247 @@ +import { Button, mergeClasses, Spinner, Tooltip } from '@fluentui/react-components'; +import { EditRegular, PlayRegular, DeleteRegular } from '@fluentui/react-icons'; +import type { Evaluator } from '@microsoft/logic-apps-shared'; +import { useSelectedRun, useSelectedAction, useCanRunEvaluation } from '../../core/state/evaluation/evaluationSelectors'; +import { useEvaluation } from '../../core/queries/evaluations'; +import { useEvaluateViewStyles } from './EvaluateView.styles'; + +interface EvaluatorDetailsPanelProps { + workflowName: string; + evaluator: Evaluator; + onEdit: () => void; + onRun: () => void; + onDelete: () => void; +} + +export const EvaluatorDetailsPanel = ({ workflowName, evaluator, onEdit, onRun, onDelete }: EvaluatorDetailsPanelProps) => { + const styles = useEvaluateViewStyles(); + const selectedRun = useSelectedRun(); + const selectedAction = useSelectedAction(); + const canRun = useCanRunEvaluation(); + + const { data: evaluation, isLoading: isEvaluationLoading } = useEvaluation( + workflowName, + selectedRun?.name ?? '', + selectedAction?.name ?? '', + evaluator.name + ); + const isEvalPassed = evaluation?.result?.toLowerCase() === 'passed'; + + return ( +
+
+
+

Evaluator Details

+

{evaluator.name}

+
+
+ +
+ {/* Evaluator Definition */} +
+
+ Evaluator name + {evaluator.name} +
+ +
+ Template + {evaluator.template === 'CustomPrompt' ? 'Custom Prompt' : evaluator.template} +
+ + {evaluator.template !== 'ToolCallTrajectory' && ( + <> + {evaluator.deploymentId && ( +
+ Deployment ID + {evaluator.deploymentId} +
+ )} + + {evaluator.agentModelType && ( +
+ Agent model type + {evaluator.agentModelType} +
+ )} + + {evaluator.modelConfiguration?.referenceName && ( +
+ Model connection reference + {evaluator.modelConfiguration.referenceName} +
+ )} + + {evaluator.agentModelSettings?.deploymentModelProperties?.name && ( +
+ Deployment model name + {evaluator.agentModelSettings.deploymentModelProperties.name} +
+ )} + + )} + + {evaluator.groundTruthRunId && ( +
+ Ground truth run ID + {evaluator.groundTruthRunId} +
+ )} + + {evaluator.groundTruthAgentActionName && ( +
+ Ground truth agent action + {evaluator.groundTruthAgentActionName} +
+ )} + + {/* Template-specific parameters */} + {evaluator.template === 'CustomPrompt' && evaluator.parameters.prompt && ( +
+ Instructions +
{evaluator.parameters.prompt}
+
+ )} + + {evaluator.template === 'ToolCallTrajectory' && ( + <> + {evaluator.parameters.expectedToolCalls && evaluator.parameters.expectedToolCalls.length > 0 && ( +
+ Expected Tool Calls +
+ {evaluator.parameters.expectedToolCalls.map((tc, idx) => ( +
+
+ {tc.name} +
+ {tc.arguments && Object.keys(tc.arguments).length > 0 && ( +
{JSON.stringify(tc.arguments, null, 2)}
+ )} +
+ ))} +
+
+ )} + + {evaluator.parameters.comparisonMethod && ( +
+ Comparison method + {evaluator.parameters.comparisonMethod} +
+ )} + + {evaluator.parameters.threshold !== undefined && ( +
+ Threshold + {evaluator.parameters.threshold} +
+ )} + +
+ Compare arguments + {evaluator.parameters.shouldCompareArgs ? 'Yes' : 'No'} +
+ + )} + + {evaluator.template === 'SemanticSimilarity' && evaluator.parameters.expectedChatResponse && ( +
+ Expected Chat Response +
{evaluator.parameters.expectedChatResponse}
+
+ )} + +
+ + + + + + + + + +
+
+ + {/* Last Evaluation Result */} + {selectedRun && ( +
+ {isEvaluationLoading ? ( +
+ +
+ ) : evaluation ? ( + <> +
+ Last Evaluation + + {evaluation.result} + +
+ +
+
+ Status + + {evaluation.result} + +
+
+ Value + {evaluation.value} +
+ {evaluation.agentActionName && ( +
+ Agent Action + {evaluation.agentActionName} +
+ )} +
+ + {evaluation.reason && ( +
+ Reason +
{evaluation.reason}
+
+ )} + +
+
+ Total Tokens + {evaluation.totalTokens} +
+
+ Input Tokens + {evaluation.inputTokens} +
+
+ Output Tokens + {evaluation.outputTokens} +
+
+ + ) : ( +
+

No evaluation results for this run

+

Click Run to evaluate

+
+ )} +
+ )} + + {!selectedRun && ( +
+

Select a run to view evaluation results

+
+ )} +
+
+ ); +}; diff --git a/libs/designer-v2/src/lib/ui/evaluation/EvaluatorFormPanel.tsx b/libs/designer-v2/src/lib/ui/evaluation/EvaluatorFormPanel.tsx new file mode 100644 index 00000000000..7e75f880998 --- /dev/null +++ b/libs/designer-v2/src/lib/ui/evaluation/EvaluatorFormPanel.tsx @@ -0,0 +1,356 @@ +import { useState, useEffect, useCallback } from 'react'; +import { Button, Dropdown, Input, Label, Option, Radio, RadioGroup, Checkbox, Textarea, Spinner } from '@fluentui/react-components'; +import { DeleteRegular, AddRegular, SaveRegular } from '@fluentui/react-icons'; +import { useDispatch } from 'react-redux'; +import { finishFormAction, cancelFormAction } from '../../core/state/evaluation/evaluationSlice'; +import { useEditingEvaluator, useRightPanelView, useSelectedAction } from '../../core/state/evaluation/evaluationSelectors'; +import { useCreateOrUpdateEvaluator } from '../../core/queries/evaluations'; +import type { EvaluatorTemplate, ComparisonMethod } from '@microsoft/logic-apps-shared'; +import type { EvaluatorFormData } from './evaluatorFormHelpers'; +import { + createDefaultEvaluatorFormData, + evaluatorToFormData, + formDataToEvaluator, + createDefaultToolCallFormItem, +} from './evaluatorFormHelpers'; +import { useEvaluateViewStyles } from './EvaluateView.styles'; + +interface EvaluatorFormPanelProps { + workflowName: string; +} + +export const EvaluatorFormPanel = ({ workflowName }: EvaluatorFormPanelProps) => { + const styles = useEvaluateViewStyles(); + const dispatch = useDispatch(); + const rightPanelView = useRightPanelView(); + const editingEvaluator = useEditingEvaluator(); + const mode = rightPanelView === 'edit' ? 'edit' : 'create'; + const selectedAction = useSelectedAction(); + const { mutateAsync: createOrUpdateEvaluator, isLoading: isModifyingEvaluator } = useCreateOrUpdateEvaluator( + workflowName, + selectedAction?.name ?? '' + ); + + const [formData, setFormData] = useState(createDefaultEvaluatorFormData()); + const [error, setError] = useState(null); + + useEffect(() => { + if (mode === 'edit' && editingEvaluator) { + setFormData(evaluatorToFormData(editingEvaluator)); + } else { + setFormData(createDefaultEvaluatorFormData()); + } + }, [mode, editingEvaluator]); + + const updateFormField = useCallback((field: K, value: EvaluatorFormData[K]) => { + setFormData((prev) => ({ ...prev, [field]: value })); + }, []); + + const handleSubmit = useCallback(async () => { + if (!formData.name.trim()) { + setError('Evaluator name is required'); + return; + } + + setError(null); + try { + const evalData = formDataToEvaluator(formData); + await createOrUpdateEvaluator({ evaluatorName: formData.name, evaluator: evalData }); + dispatch(finishFormAction()); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to save evaluator'); + } + }, [formData, createOrUpdateEvaluator, dispatch]); + + return ( +
+
+
+

{mode === 'create' ? 'Create Evaluator' : 'Edit Evaluator'}

+

+ {mode === 'create' ? 'Configure a new evaluator for your agent' : 'Update evaluator configuration'} +

+
+
+ + {error && ( +
+ {error} + +
+ )} + +
+
+ + updateFormField('name', data.value)} + placeholder="Enter evaluator name" + disabled={mode === 'edit'} + style={{ width: '100%' }} + /> +
+ +
+ + updateFormField('template', data.optionValue as EvaluatorTemplate)} + style={{ width: '100%' }} + > + + + + +
+ + {/* Model configuration - not needed for ToolCallTrajectory */} + {formData.template !== 'ToolCallTrajectory' && ( + <> +
+ + updateFormField('agentModelType', data.optionValue as string)} + style={{ width: '100%' }} + > + + + +
+ +
+ + updateFormField('deploymentId', data.value)} + placeholder="Enter deployment identifier" + style={{ width: '100%' }} + /> +
+ +
+ + updateFormField('modelReferenceName', data.value)} + placeholder="Enter model connection reference name" + style={{ width: '100%' }} + /> +
+ +
+ + updateFormField('modelName', data.value)} + placeholder="Enter deployment model name" + style={{ width: '100%' }} + /> +
+ + )} + + {/* Ground truth fields */} + {(formData.template === 'ToolCallTrajectory' || formData.template === 'SemanticSimilarity') && ( + <> + updateFormField('useGroundTruthRun', data.value === 'groundTruth')} + > + + + + + {formData.useGroundTruthRun && ( + <> +
+ + updateFormField('groundTruthRunId', data.value)} + placeholder="Enter ground truth run identifier" + style={{ width: '100%' }} + /> +
+
+ + updateFormField('groundTruthAgentActionName', data.value)} + placeholder="Enter ground truth agent action name" + style={{ width: '100%' }} + /> +
+ + )} + + )} + + {/* CustomPrompt: Instructions */} + {formData.template === 'CustomPrompt' && ( +
+ +