| // Copyright 2024 The Chromium Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| import * as Host from '../../../core/host/host.js'; |
| import type * as Platform from '../../../core/platform/platform.js'; |
| import * as Root from '../../../core/root/root.js'; |
| import * as SDK from '../../../core/sdk/sdk.js'; |
| import type * as TextUtils from '../../../core/text_utils/text_utils.js'; |
| import type * as Protocol from '../../../generated/protocol.js'; |
| import type * as LHModel from '../../lighthouse/lighthouse.js'; |
| import type * as Trace from '../../trace/trace.js'; |
| import type * as Workspace from '../../workspace/workspace.js'; |
| import {debugLog, isStructuredLogEnabled} from '../debug.js'; |
| import {dispatchAiAssistanceDoneEvent} from '../DOMHelpers.js'; |
| import { |
| type ContextHandlerResult, |
| type DataHandlerResult, |
| PermissionDecision, |
| type PermissionPrompt, |
| } from '../tools/Tool.js'; |
| |
| type UrlString = Platform.DevToolsPath.UrlString; |
| const MAX_SUGGESTION_LENGTH = 200; |
| |
| export const enum ResponseType { |
| CONTEXT = 'context', |
| TITLE = 'title', |
| THOUGHT = 'thought', |
| ACTION = 'action', |
| SIDE_EFFECT = 'side-effect', |
| SUGGESTIONS = 'suggestions', |
| ANSWER = 'answer', |
| ERROR = 'error', |
| QUERYING = 'querying', |
| USER_QUERY = 'user-query', |
| CONTEXT_CHANGE = 'context-change', |
| } |
| |
| export const enum ErrorType { |
| UNKNOWN = 'unknown', |
| ABORT = 'abort', |
| MAX_STEPS = 'max-steps', |
| BLOCK = 'block', |
| CROSS_ORIGIN = 'cross-origin', |
| QUOTA = 'quota', |
| PAYLOAD_TOO_LARGE = 'payload-too-large', |
| } |
| |
| export const enum MultimodalInputType { |
| SCREENSHOT = 'screenshot', |
| UPLOADED_IMAGE = 'uploaded-image', |
| } |
| |
| export interface MultimodalInput { |
| input: Host.AidaClient.Part; |
| type: MultimodalInputType; |
| id: string; |
| } |
| |
| export interface AnswerResponse { |
| type: ResponseType.ANSWER; |
| text: string; |
| // Whether this is the complete answer or only a part of it (for streaming reasons) |
| complete: boolean; |
| rpcId?: Host.AidaClient.RpcGlobalId; |
| suggestions?: [string, ...string[]]; |
| widgets?: AiWidget[]; |
| } |
| |
| export interface SuggestionsResponse { |
| type: ResponseType.SUGGESTIONS; |
| suggestions: [string, ...string[]]; |
| } |
| |
| export interface ErrorResponse { |
| type: ResponseType.ERROR; |
| error: ErrorType; |
| } |
| |
| export interface ContextDetail { |
| title: string; |
| text: string; |
| codeLang?: string; |
| } |
| export interface ContextResponse { |
| type: ResponseType.CONTEXT; |
| details: [ContextDetail, ...ContextDetail[]]; |
| widgets?: AiWidget[]; |
| } |
| |
| export interface TitleResponse { |
| type: ResponseType.TITLE; |
| title: string; |
| rpcId?: Host.AidaClient.RpcGlobalId; |
| } |
| |
| export interface ThoughtResponse { |
| type: ResponseType.THOUGHT; |
| thought: string; |
| rpcId?: Host.AidaClient.RpcGlobalId; |
| } |
| |
| export interface SideEffectResponse { |
| type: ResponseType.SIDE_EFFECT; |
| description: string|null; |
| code?: string; |
| confirm: (decision: PermissionDecision) => void; |
| permissionPrompt?: PermissionPrompt; |
| permissionTitle?: string; |
| } |
| export interface ContextChangeResponse { |
| type: ResponseType.CONTEXT_CHANGE; |
| /** |
| * Information to pass down what was selected |
| * Use to make the LLM understand the the user |
| * already selected something. |
| */ |
| description: string; |
| context: ConversationContext<unknown>; |
| widgets?: AiWidget[]; |
| } |
| |
| interface SerializedSideEffectResponse extends Omit<SideEffectResponse, 'confirm'> {} |
| |
| export interface ActionResponse { |
| type: ResponseType.ACTION; |
| code?: string; |
| output?: string; |
| canceled: boolean; |
| widgets?: AiWidget[]; |
| /** |
| * The name of the executed tool. Only populated for AI v2. |
| */ |
| toolName?: string; |
| } |
| |
| export interface QueryingResponse { |
| type: ResponseType.QUERYING; |
| } |
| |
| export interface UserQuery { |
| type: ResponseType.USER_QUERY; |
| query: string; |
| imageInput?: Host.AidaClient.Part; |
| imageId?: string; |
| } |
| |
| export type ResponseData = AnswerResponse|SuggestionsResponse|ErrorResponse|ActionResponse|SideEffectResponse| |
| ThoughtResponse|TitleResponse|QueryingResponse|ContextResponse|UserQuery|ContextChangeResponse; |
| |
| export type SerializedResponseData = AnswerResponse|SuggestionsResponse|ErrorResponse|ActionResponse| |
| SerializedSideEffectResponse|ThoughtResponse|TitleResponse|QueryingResponse|ContextResponse|UserQuery; |
| |
| export type FunctionCallResponseData = |
| TitleResponse|ThoughtResponse|ActionResponse|SideEffectResponse|SuggestionsResponse|ContextChangeResponse; |
| |
| export interface BuildRequestOptions { |
| text: string; |
| } |
| |
| export interface RequestOptions { |
| temperature?: number; |
| modelId?: string; |
| } |
| |
| export type AllowedOriginResult = { |
| origin: SDK.SecurityOrigin.SecurityOrigin|undefined, |
| }|{ |
| blocked: true, |
| }; |
| |
| export interface AgentOptions { |
| aidaClient: Host.AidaClient.AidaClient; |
| serverSideLoggingAllowed?: boolean; |
| sessionId?: string; |
| confirmSideEffectForTest?: typeof Promise.withResolvers; |
| onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>; |
| history?: Host.AidaClient.Content[]; |
| allowedOrigin?: () => AllowedOriginResult; |
| lighthouseRecording?: (overrides?: LHModel.RunTypes.RunOverrides) => Promise<LHModel.ReporterTypes.ReportJSON|null>; |
| targetManager?: SDK.TargetManager.TargetManager; |
| } |
| |
| export interface ParsedAnswer { |
| answer: string; |
| suggestions?: [string, ...string[]]; |
| } |
| |
| export type ParsedResponse = ParsedAnswer; |
| |
| export const MAX_STEPS = 20; |
| |
| export interface ConversationSuggestion { |
| title: string; |
| jslogContext?: string; |
| } |
| |
| /** At least one. */ |
| export type ConversationSuggestions = [ConversationSuggestion, ...ConversationSuggestion[]]; |
| |
| export type ConversationContextJslog = 'ai-context-dom-node'|'ai-context-network-request'|'ai-context-file'| |
| 'ai-context-performance-trace'|'ai-context-accessibility'|'ai-context-storage'; |
| |
| export abstract class ConversationContext<T> { |
| abstract readonly jslogContext: ConversationContextJslog; |
| abstract getItem(): T; |
| abstract getTitle(): string; |
| |
| /** |
| * Returns true if the server-side logging is enabled when this context is active. |
| * Currently only used for AI v2. |
| */ |
| isLoggingEnabled(): boolean { |
| return true; |
| } |
| |
| /** |
| * Returns the security origin that owns this context data. |
| * |
| * The AI Assistance panel locks each conversation to the origin of the initial |
| * context. If the user selects a context with a different origin, DevTools |
| * blocks access to prevent unauthorized cross-origin data exposure. |
| * |
| * Subclasses must implement this method. If a context is detached, invalid, |
| * or anonymous, the method must return a unique opaque origin |
| * (`SDK.SecurityOrigin.SecurityOrigin.createUniqueOpaque()`). |
| * |
| * @returns The {@link SDK.SecurityOrigin.SecurityOrigin} that owns this context. |
| */ |
| abstract getOrigin(): SDK.SecurityOrigin.SecurityOrigin; |
| |
| /** |
| * Checks whether this context can participate in a conversation locked to `establishedOrigin`. |
| * |
| * Evaluation rules: |
| * 1. Returns `false` if this context origin is opaque. Opaque contexts can never |
| * participate in AI conversations. |
| * 2. Returns `true` if `establishedOrigin` is `undefined` (conversation is not yet locked). |
| * 3. Returns `true` if this context origin is same-origin with `establishedOrigin`. |
| * |
| * @param establishedOrigin The locked origin of the current conversation, or `undefined` |
| * if the conversation has not made its first query. |
| */ |
| isOriginAllowed(establishedOrigin: SDK.SecurityOrigin.SecurityOrigin|undefined): boolean { |
| const origin = this.getOrigin(); |
| |
| if (origin.isOpaque()) { |
| return false; |
| } |
| if (!establishedOrigin) { |
| return true; |
| } |
| return origin.isSameOriginWith(establishedOrigin); |
| } |
| |
| /** |
| * This method is called at the start of `AiAgent.run`. |
| * It will be overridden in subclasses to fetch data related to the context item. |
| */ |
| async refresh(): Promise<void> { |
| return; |
| } |
| |
| async getSuggestions(): Promise<ConversationSuggestions|undefined> { |
| return; |
| } |
| |
| /** |
| * Returns a detailed description of the context item for inclusion in the AI model prompt. |
| * Currently only used by AiAgent2. |
| */ |
| async getPromptDetails(): Promise<string|null> { |
| return null; |
| } |
| |
| /** |
| * Returns a list of context details to display to the user in the UI. |
| * Currently only used by AiAgent2. |
| */ |
| async getUserFacingDetails(): Promise<[ContextDetail, ...ContextDetail[]]|null> { |
| return null; |
| } |
| |
| /** |
| * Returns initial UI widgets to display in the conversation context header |
| * when this context is active (e.g. Core Web Vitals summary for a performance trace). |
| * Used by PerformanceAgent and AiAgent2. |
| */ |
| async getWidgets(): Promise<AiWidget[]> { |
| return []; |
| } |
| } |
| |
| export interface ComputedStyleAiWidget { |
| name: 'COMPUTED_STYLES'; |
| data: { |
| computedStyles: Map<string, string>, |
| backendNodeId: Protocol.DOM.BackendNodeId, |
| matchedCascade: SDK.CSSMatchedStyles.CSSMatchedStyles, |
| // The subset of CSS properties that the AI looked up. |
| properties: string[], |
| }; |
| } |
| export interface CoreVitalsAiWidget { |
| name: 'CORE_VITALS'; |
| data: { |
| insightSetKey: string, |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| }; |
| } |
| |
| export interface StylePropertiesAiWidget { |
| name: 'STYLE_PROPERTIES'; |
| data: { |
| backendNodeId: Protocol.DOM.BackendNodeId, |
| selector?: string, |
| }; |
| } |
| |
| export interface DomTreeAiWidget { |
| name: 'DOM_TREE'; |
| data: { |
| root: SDK.DOMModel.DOMNodeSnapshot, |
| title: Platform.UIString.LocalizedString, |
| accessibleRevealLabel: Platform.UIString.LocalizedString, |
| networkRequest?: { |
| url: string, |
| size: number, |
| resourceType: Protocol.Network.ResourceType, |
| mimeType: string, |
| imageContent?: TextUtils.ContentData.ContentData, |
| }, |
| }; |
| } |
| |
| export interface PerformanceTraceAiWidget { |
| name: 'PERFORMANCE_TRACE'; |
| data: { |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| }; |
| } |
| |
| export interface PerfInsightAiWidget { |
| name: 'PERF_INSIGHT'; |
| data: { |
| insight: Trace.Insights.Types.InsightKeys, |
| insightData: Trace.Insights.Types.InsightModel, |
| }; |
| } |
| |
| export interface TimelineRangeSummaryAiWidget { |
| name: 'TIMELINE_RANGE_SUMMARY'; |
| data: { |
| bounds: Trace.Types.Timing.TraceWindowMicro, |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| // We can use this component for other tracks summaries later, so |
| // we include the track in the data. |
| track: 'main', |
| }; |
| } |
| |
| export interface BottomUpTreeAiWidget { |
| name: 'BOTTOM_UP_TREE'; |
| data: { |
| bounds: Trace.Types.Timing.TraceWindowMicro, |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| }; |
| } |
| |
| export interface SourceFileAiWidget { |
| name: 'SOURCE_FILE'; |
| data: { |
| uiSourceCode: Workspace.UISourceCode.UISourceCode, |
| }; |
| } |
| |
| export interface SourceFilesListAiWidget { |
| name: 'SOURCE_FILES_LIST'; |
| data: { |
| uiSourceCodes: Workspace.UISourceCode.UISourceCode[], |
| }; |
| } |
| |
| export interface NetworkRequestsListAiWidget { |
| name: 'NETWORK_REQUESTS_LIST'; |
| data: { |
| requests: SDK.NetworkRequest.NetworkRequest[], |
| }; |
| } |
| export interface NetworkTrackAiWidget { |
| name: 'NETWORK_TRACK'; |
| data: { |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| bounds: Trace.Types.Timing.TraceWindowMicro, |
| }; |
| } |
| |
| export interface LighthouseReportAiWidget { |
| name: 'LIGHTHOUSE_REPORT'; |
| data: { |
| report: LHModel.ReporterTypes.ReportJSON, |
| snapshotReport?: boolean, |
| }; |
| } |
| |
| export interface TimelineEventSummaryAiWidget { |
| name: 'TIMELINE_EVENT_SUMMARY'; |
| data: { |
| event: Trace.Types.Events.Event, |
| parsedTrace: Trace.TraceModel.ParsedTrace, |
| }; |
| } |
| |
| export interface NetworkRequestGeneralHeadersAiWidget { |
| name: 'NETWORK_REQUEST_GENERAL_HEADERS'; |
| data: { |
| request: SDK.NetworkRequest.NetworkRequest, |
| }; |
| } |
| |
| // If the line and column are not provided, we assume the whole file was sent to the agent.s |
| export interface SourceCodeAiWidget { |
| name: 'SOURCE_CODE'; |
| data: { |
| url: UrlString, |
| code: string, |
| line?: number, |
| column?: number, |
| }; |
| } |
| |
| export interface StorageBreakdownAiWidget { |
| name: 'STORAGE_BREAKDOWN'; |
| data: { |
| totalUsageBytes: number, |
| totalQuotaBytes: number, |
| usageBreakdown: Array<{ |
| storageType: string, |
| bytes: number, |
| }>, |
| }; |
| } |
| |
| export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget|DomTreeAiWidget| |
| PerformanceTraceAiWidget|PerfInsightAiWidget|TimelineRangeSummaryAiWidget|BottomUpTreeAiWidget|SourceFileAiWidget| |
| LighthouseReportAiWidget|TimelineEventSummaryAiWidget|NetworkRequestGeneralHeadersAiWidget|SourceCodeAiWidget| |
| SourceFilesListAiWidget|NetworkRequestsListAiWidget|NetworkTrackAiWidget|StorageBreakdownAiWidget; |
| |
| /** |
| * @deprecated Used for v1 agents. Once v2 is shipped, this can be removed. |
| * Use `DataHandlerResult` or `ContextHandlerResult` from `Tool.js` instead. |
| */ |
| export type ToolResult<ResultType> = DataHandlerResult<ResultType>|ContextHandlerResult; |
| |
| export interface FunctionHandlerOptions { |
| /** |
| * Shows that the user approved |
| * the execution if it was required |
| */ |
| approved?: boolean; |
| signal?: AbortSignal; |
| } |
| |
| export interface FunctionDeclaration<Args extends Record<string, unknown>, ReturnType> { |
| /** |
| * Description of function, this is send to the LLM |
| * to explain what will the function do. |
| */ |
| description: string|(() => string); |
| /** |
| * JSON schema like representation of the parameters |
| * the function needs to be called with. |
| * Provide description to all parameters as this is |
| * send to the LLM. |
| */ |
| parameters: Host.AidaClient.FunctionObjectParam<keyof Args>; |
| /** |
| * Provided a way to give information back to the UI. |
| */ |
| displayInfoFromArgs?: ( |
| args: Args, |
| ) => { |
| title?: string, thought?: string, action?: string, suggestions?: [string, ...string[]], |
| }; |
| /** |
| * Choices the permission prompt offers when the handler returns |
| * `requiresApproval`. Behaves as `ALLOW_ONCE` when unset. |
| */ |
| permissionPrompt?: PermissionPrompt; |
| /** |
| * Title of the permission prompt, e.g. "Allow reading cookie values?". |
| */ |
| permissionTitle?: string; |
| /** |
| * Function implementation that the LLM will try to execute, |
| */ |
| handler(args: Args, options?: FunctionHandlerOptions): Promise<ToolResult<ReturnType>>; |
| } |
| |
| interface AidaFetchResult { |
| text?: string; |
| functionCall?: Host.AidaClient.AidaFunctionCall; |
| completed: boolean; |
| rpcId?: Host.AidaClient.RpcGlobalId; |
| } |
| |
| class CrossOriginError extends Error { |
| constructor() { |
| super('Cross-origin navigation detected'); |
| this.name = 'CrossOriginError'; |
| } |
| } |
| |
| /** |
| * AiAgent is a base class for implementing an interaction with AIDA |
| * that involves one or more requests being sent to AIDA optionally |
| * utilizing function calling. |
| * |
| * TODO: missing a test that action code is yielded before the |
| * confirmation dialog. |
| */ |
| export abstract class AiAgent<T> { |
| /** |
| * WARNING: preamble defined in code is only used when userTier is |
| * TESTERS. Otherwise, a server-side preamble is used (see |
| * chrome_preambles.gcl). |
| */ |
| abstract readonly preamble: string|undefined; |
| abstract readonly options: RequestOptions; |
| abstract readonly clientFeature: Host.AidaClient.ClientFeature; |
| abstract readonly userTier: string|undefined; |
| abstract handleContextDetails(select: ConversationContext<T>|null): AsyncGenerator<ContextResponse, void, void>; |
| |
| readonly #sessionId: string; |
| readonly #aidaClient: Host.AidaClient.AidaClient; |
| /** |
| * Tracks the dynamic runtime state of logging. Even if logging is allowed |
| * by policy, tools or sensitive contexts can deactivate this to avoid logging sensitive data. |
| */ |
| #serverSideLoggingActive: boolean; |
| readonly confirmSideEffect: typeof Promise.withResolvers; |
| readonly #functionDeclarations = new Map<string, FunctionDeclaration<Record<string, unknown>, unknown>>(); |
| readonly #allowedOrigin?: () => AllowedOriginResult; |
| readonly #targetManager: SDK.TargetManager.TargetManager; |
| |
| /** |
| * Used in the debug mode and evals. |
| */ |
| readonly #structuredLog: Array<{ |
| request: Host.AidaClient.DoConversationRequest, |
| aidaResponse: Host.AidaClient.DoConversationResponse, |
| }> = []; |
| |
| /** |
| * `context` does not change during `AiAgent.run()`, ensuring that calls to JS |
| * have the correct `context`. We don't want element selection by the user to |
| * change the `context` during an `AiAgent.run()`. |
| */ |
| protected context?: ConversationContext<T>; |
| |
| #history: Host.AidaClient.Content[]; |
| |
| #facts: Set<Host.AidaClient.RequestFact> = new Set<Host.AidaClient.RequestFact>(); |
| |
| constructor(opts: AgentOptions) { |
| this.#aidaClient = opts.aidaClient; |
| let serverSideLoggingAllowed = opts.serverSideLoggingAllowed ?? false; |
| // Disable logging for now. |
| // For context, see b/454563259#comment35. |
| // We should be able to remove this ~end of April. |
| if (Root.Runtime.hostConfig.devToolsGeminiRebranding?.enabled) { |
| serverSideLoggingAllowed = false; |
| } |
| this.#serverSideLoggingActive = serverSideLoggingAllowed; |
| this.#sessionId = opts.sessionId ?? crypto.randomUUID(); |
| this.confirmSideEffect = opts.confirmSideEffectForTest ?? (() => Promise.withResolvers()); |
| this.#history = opts.history ?? []; |
| this.#allowedOrigin = opts.allowedOrigin; |
| // eslint-disable-next-line @devtools/no-instance-of-migrated-singletons |
| this.#targetManager = opts.targetManager ?? SDK.TargetManager.TargetManager.instance(); |
| } |
| |
| async enhanceQuery(query: string, selected: ConversationContext<T>|null, |
| multimodalInputType?: MultimodalInputType): Promise<string>; |
| async enhanceQuery(query: string): Promise<string> { |
| return query; |
| } |
| |
| currentFacts(): ReadonlySet<Host.AidaClient.RequestFact> { |
| return this.#facts; |
| } |
| |
| get history(): Host.AidaClient.Content[] { |
| return [...this.#history]; |
| } |
| |
| get targetManager(): SDK.TargetManager.TargetManager { |
| return this.#targetManager; |
| } |
| |
| /** |
| * Add a fact which will be sent for any subsequent requests. |
| * Returns the new list of all facts. |
| * Facts are never automatically removed. |
| */ |
| addFact(fact: Host.AidaClient.RequestFact): ReadonlySet<Host.AidaClient.RequestFact> { |
| this.#facts.add(fact); |
| return this.#facts; |
| } |
| |
| removeFact(fact: Host.AidaClient.RequestFact): boolean { |
| return this.#facts.delete(fact); |
| } |
| |
| clearFacts(): void { |
| this.#facts.clear(); |
| } |
| |
| /** |
| * Clears any subclass-specific caches. This is called when a run encounters |
| * an error (e.g., cross-origin navigation, abort, or execution error) to |
| * prevent unvalidated cached data from being replayed in subsequent runs. |
| */ |
| clearCache(): void { |
| } |
| |
| /** |
| * Disables server-side logging for the remainder of this agent instance's lifetime. |
| * |
| * Logging deactivation is irreversible for the session. Conversation history |
| * accumulates across turns; re-enabling logging later would leak sensitive |
| * data from prior turns to AIDA. |
| */ |
| protected disableServerSideLogging(): void { |
| this.#serverSideLoggingActive = false; |
| } |
| |
| popPendingMultimodalInput(): MultimodalInput|undefined { |
| return undefined; |
| } |
| |
| /** |
| * Preamble features appended to the `client_version` in metadata. |
| * This is required ONLY for the Styling Agent for legacy reasons to serve |
| * different server-side preambles based on the Chrome version. |
| * Other agents should NOT set or override this. |
| * If you are curious about this, look for `do_conversation_handler.cc` in |
| * Google3 or chat to @jacktfranklin. |
| */ |
| preambleFeatures(): string[] { |
| return []; |
| } |
| |
| buildRequest(part: Host.AidaClient.Part|Host.AidaClient.Part[], |
| role: Host.AidaClient.Role.USER| |
| Host.AidaClient.Role.ROLE_UNSPECIFIED): Host.AidaClient.DoConversationRequest { |
| const parts = Array.isArray(part) ? part : [part]; |
| const currentMessage: Host.AidaClient.Content = { |
| parts, |
| role, |
| }; |
| const history = [...this.#history]; |
| const declarations: Host.AidaClient.FunctionDeclaration[] = []; |
| for (const [name, definition] of this.#functionDeclarations.entries()) { |
| declarations.push({ |
| name, |
| description: typeof definition.description === 'function' ? definition.description() : definition.description, |
| parameters: definition.parameters, |
| }); |
| } |
| function validTemperature(temperature: number|undefined): number|undefined { |
| return typeof temperature === 'number' && temperature >= 0 ? temperature : undefined; |
| } |
| const enableAidaFunctionCalling = declarations.length; |
| const userTier = Host.AidaClient.convertToUserTierEnum(this.userTier); |
| const preamble = userTier === Host.AidaClient.UserTier.TESTERS ? this.preamble : undefined; |
| const facts = Array.from(this.#facts); |
| const request: Host.AidaClient.DoConversationRequest = { |
| client: Host.AidaClient.CLIENT_NAME, |
| current_message: currentMessage, |
| preamble, |
| |
| historical_contexts: history.length ? history : undefined, |
| facts: facts.length ? facts : undefined, |
| |
| ...(enableAidaFunctionCalling ? {function_declarations: declarations} : {}), |
| options: { |
| temperature: validTemperature(this.options.temperature), |
| model_id: this.options.modelId || undefined, |
| }, |
| metadata: { |
| disable_user_content_logging: !(this.#serverSideLoggingActive ?? false), |
| string_session_id: this.#sessionId, |
| user_tier: userTier, |
| client_version: |
| Root.Runtime.getChromeVersion() + this.preambleFeatures().map(feature => `+${feature}`).join(''), |
| }, |
| |
| functionality_type: enableAidaFunctionCalling ? Host.AidaClient.FunctionalityType.AGENTIC_CHAT : |
| Host.AidaClient.FunctionalityType.CHAT, |
| |
| client_feature: this.clientFeature, |
| }; |
| return request; |
| } |
| |
| get sessionId(): string { |
| return this.#sessionId; |
| } |
| |
| /** |
| * The AI has instructions to emit structured suggestions in their response. This |
| * function parses for that. |
| * |
| * Note: currently only StylingAgent and PerformanceAgent utilize this, but |
| * eventually all agents should support this. |
| */ |
| parseTextResponseForSuggestions(text: string): ParsedResponse { |
| if (!text) { |
| return {answer: ''}; |
| } |
| |
| const lines = text.split('\n'); |
| const answerLines: string[] = []; |
| let suggestions: [string, ...string[]]|undefined; |
| |
| for (const line of lines) { |
| const trimmed = line.trim(); |
| if (trimmed.startsWith('SUGGESTIONS:')) { |
| try { |
| suggestions = sanitizeSuggestions(trimmed.substring('SUGGESTIONS:'.length).trim()); |
| } catch { |
| } |
| } else { |
| answerLines.push(line); |
| } |
| } |
| |
| // Sometimes the model fails to put the SUGGESTIONS text on its own line. Handle |
| // the case where the suggestions are part of the last line of the answer. |
| if (!suggestions && answerLines.at(-1)?.includes('SUGGESTIONS:')) { |
| const [answer, suggestionsText] = answerLines[answerLines.length - 1].split('SUGGESTIONS:', 2); |
| try { |
| suggestions = sanitizeSuggestions(suggestionsText.trim()); |
| } catch { |
| } |
| answerLines[answerLines.length - 1] = answer; |
| } |
| |
| const response: ParsedResponse = { |
| // If we could not parse the parts, consider the response to be an |
| // answer. |
| answer: answerLines.join('\n'), |
| }; |
| |
| if (suggestions) { |
| response.suggestions = suggestions; |
| } |
| |
| return response; |
| } |
| |
| /** |
| * Parses a streaming text response into a |
| * though/action/title/answer/suggestions component. |
| */ |
| parseTextResponse(response: string): ParsedResponse { |
| return this.parseTextResponseForSuggestions(response.trim()); |
| } |
| |
| /** |
| * Parses the text of a response that is still streaming. Only the `answer` |
| * of the result is shown, as a partial answer. By default, partial answers |
| * use the same parsing as completed answers. |
| * |
| * This hook exists so that `AiAgent2` (AI V2) can parse follow-up |
| * suggestions differently without changing the V1 agents. Remove it once |
| * AI V2 ships and the V1 agents are removed. b/568697679 explores getting |
| * suggestions from a function call instead of parsing them from text, |
| * which would remove the need for this parsing. |
| */ |
| protected parsePartialTextResponse(response: string): ParsedResponse { |
| return this.parseTextResponse(response); |
| } |
| |
| protected async finalizeAnswer(answer: AnswerResponse): Promise<AnswerResponse> { |
| return answer; |
| } |
| |
| /** |
| * Declare a function that the AI model can call. |
| * @param name The name of the function |
| * @param declaration the function declaration. Currently functions must: |
| * 1. Return an object of serializable key/value pairs. You cannot return |
| * anything other than a plain JavaScript object that can be serialized. |
| * 2. Take one parameter which is an object that can have |
| * multiple keys and values. For example, rather than a function being called |
| * with two args, `foo` and `bar`, you should instead have the function be |
| * called with one object with `foo` and `bar` keys. |
| */ |
| protected declareFunction<Args extends Record<string, unknown>, ReturnType = unknown>( |
| name: string, |
| declaration: FunctionDeclaration<Args, ReturnType>, |
| ): void { |
| if (this.#functionDeclarations.has(name)) { |
| throw new Error(`Duplicate function declaration ${name}`); |
| } |
| this.#functionDeclarations.set(name, declaration as FunctionDeclaration<Record<string, unknown>, ReturnType>); |
| } |
| |
| protected clearDeclaredFunctions(): void { |
| this.#functionDeclarations.clear(); |
| } |
| |
| /** |
| * Executed immediately after the current context is populated with the selected |
| * context and before the request is built. |
| */ |
| protected async preRun(): Promise<void> { |
| } |
| |
| async * |
| run( |
| initialQuery: string, |
| options: { |
| selected: ConversationContext<T>|null, |
| signal?: AbortSignal, |
| }, |
| multimodalInput?: MultimodalInput, |
| ): AsyncGenerator<ResponseData, void, void> { |
| await options.selected?.refresh(); |
| // Reset context on each run so cleared selections (`null`) do not leave |
| // stale context references on long-lived agent instances (e.g. `AiAgent2`). |
| this.context = options.selected ?? undefined; |
| await this.preRun(); |
| |
| const enhancedQuery = await this.enhanceQuery(initialQuery, options.selected, multimodalInput?.type); |
| if (!enhancedQuery.trim() && !multimodalInput) { |
| return; |
| } |
| Host.userMetrics.freestylerQueryLength(enhancedQuery.length); |
| |
| let query: Host.AidaClient.Part|Host.AidaClient.Part[]; |
| query = multimodalInput ? [{text: enhancedQuery}, multimodalInput.input] : [{text: enhancedQuery}]; |
| // Request is built here to capture history up to this point. |
| let request = this.buildRequest(query, Host.AidaClient.Role.USER); |
| |
| const clientFeatureName = Host.AidaClient.getClientFeatureName(this.clientFeature); |
| debugLog(`[AiAgent] Starting conversation with client ${clientFeatureName}, userTier ${this.userTier}`); |
| |
| yield* this.handleContextDetails(options.selected); |
| |
| for (let i = 0; i < MAX_STEPS; i++) { |
| yield { |
| type: ResponseType.QUERYING, |
| }; |
| |
| if (i === 0) { |
| debugLog('[AiAgent] Step 1: Sending user prompt to model:', enhancedQuery); |
| } else if (!Array.isArray(query) && 'functionResponse' in query) { |
| debugLog(`[AiAgent] Step ${i + 1}: Sending function response for '${query.functionResponse.name}' to model:`, |
| query.functionResponse.response); |
| } else { |
| debugLog(`[AiAgent] Step ${i + 1}: Sending request to model:`, request.current_message); |
| } |
| |
| let rpcId: Host.AidaClient.RpcGlobalId|undefined; |
| let textResponse = ''; |
| let functionCall: Host.AidaClient.AidaFunctionCall|undefined = undefined; |
| try { |
| for await (const fetchResult of this.#aidaFetch(request, {signal: options.signal})) { |
| rpcId = fetchResult.rpcId; |
| textResponse = fetchResult.text ?? ''; |
| functionCall = fetchResult.functionCall; |
| |
| if (!functionCall && !fetchResult.completed) { |
| const parsed = this.parsePartialTextResponse(textResponse); |
| const partialAnswer = 'answer' in parsed ? parsed.answer : ''; |
| if (!partialAnswer) { |
| continue; |
| } |
| // Only yield partial responses here and do not add partial answers to the history. |
| yield { |
| type: ResponseType.ANSWER, |
| text: partialAnswer, |
| complete: false, |
| }; |
| } |
| } |
| } catch (err) { |
| debugLog('Error calling the AIDA API', err); |
| const error = aidaErrorToErrorType(err); |
| yield this.#createErrorResponse(error); |
| |
| break; |
| } |
| |
| this.#history.push(request.current_message); |
| |
| if (textResponse) { |
| const parsedResponse = this.parseTextResponse(textResponse); |
| if (!('answer' in parsedResponse)) { |
| throw new Error('Expected a completed response to have an answer'); |
| } |
| if (!functionCall) { |
| debugLog(`[AiAgent] Step ${i + 1}: Model returned text response:`, parsedResponse.answer); |
| this.#history.push({ |
| parts: [{ |
| text: parsedResponse.answer, |
| }], |
| role: Host.AidaClient.Role.MODEL, |
| }); |
| } |
| Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceAnswerReceived); |
| yield await this.finalizeAnswer({ |
| type: ResponseType.ANSWER, |
| text: parsedResponse.answer, |
| suggestions: parsedResponse.suggestions, |
| complete: true, |
| rpcId, |
| }); |
| if (!functionCall) { |
| break; |
| } |
| } |
| |
| if (functionCall) { |
| debugLog(`[AiAgent] Step ${i + 1}: Model requested function call: ${functionCall.name}`, functionCall.args); |
| const allowedOriginResult = this.#allowedOrigin?.(); |
| if (allowedOriginResult && 'blocked' in allowedOriginResult) { |
| // Abort immediately if the page navigated before we could lock the origin. |
| // This prevents the AI from accessing data from the new page. |
| yield this.#createErrorResponse(ErrorType.CROSS_ORIGIN); |
| break; |
| } |
| try { |
| const result = yield* |
| this.#callFunction( |
| functionCall.name, |
| functionCall.args, |
| functionCall.thoughtSignature, |
| { |
| ...options, |
| explanation: textResponse, |
| }, |
| ); |
| |
| if (options.signal?.aborted) { |
| yield this.#createErrorResponse(ErrorType.ABORT); |
| break; |
| } |
| |
| if ('context' in result) { |
| // Pair the functionCall with a functionResponse so history stays valid |
| // if this agent instance is re-run after the context change (AiAgent2). |
| // The new context reaches the model through the next USER query. |
| this.#history.push({ |
| parts: [{ |
| functionResponse: { |
| name: functionCall.name, |
| response: {result: result.description}, |
| }, |
| }], |
| role: Host.AidaClient.Role.ROLE_UNSPECIFIED, |
| }); |
| yield { |
| type: ResponseType.CONTEXT_CHANGE, |
| description: result.description, |
| context: result.context, |
| widgets: result.widgets, |
| }; |
| |
| return; |
| } |
| |
| query = { |
| functionResponse: { |
| name: functionCall.name, |
| // Widgets are not sent back to the LLM |
| response: {...result, widgets: undefined}, |
| }, |
| }; |
| request = this.buildRequest(query, Host.AidaClient.Role.ROLE_UNSPECIFIED); |
| } catch (err) { |
| if (err instanceof CrossOriginError) { |
| yield this.#createErrorResponse(ErrorType.CROSS_ORIGIN); |
| break; |
| } |
| debugLog('Error handling function call', err); |
| yield this.#createErrorResponse(ErrorType.UNKNOWN); |
| break; |
| } |
| } else { |
| yield this.#createErrorResponse(ErrorType.UNKNOWN); |
| break; |
| } |
| |
| if (i === MAX_STEPS - 1) { |
| yield this.#createErrorResponse(ErrorType.MAX_STEPS); |
| break; |
| } |
| } |
| |
| if (isStructuredLogEnabled()) { |
| dispatchAiAssistanceDoneEvent(); |
| } |
| return; |
| } |
| |
| async * |
| #callFunction( |
| name: string, |
| args: Record<string, unknown>, |
| thoughtSignature?: string, |
| options?: FunctionHandlerOptions&{explanation?: string}, |
| ): AsyncGenerator<FunctionCallResponseData, { |
| result: unknown, |
| widgets?: AiWidget[], |
| }|{context: ConversationContext<unknown>, description: string, widgets?: AiWidget[]}> { |
| const call = this.#functionDeclarations.get(name); |
| if (!call) { |
| throw new Error(`Function ${name} is not found.`); |
| } |
| debugLog(`[AiAgent] Executing tool '${name}' with args:`, args); |
| const parts: Host.AidaClient.Part[] = []; |
| if (options?.explanation) { |
| parts.push({ |
| text: options.explanation, |
| }); |
| } |
| const functionCall: Host.AidaClient.AidaFunctionCall = { |
| name, |
| args, |
| }; |
| if (thoughtSignature) { |
| functionCall.thoughtSignature = thoughtSignature; |
| } |
| parts.push({functionCall}); |
| this.#history.push({ |
| parts, |
| role: Host.AidaClient.Role.MODEL, |
| }); |
| |
| let code; |
| if (call.displayInfoFromArgs) { |
| const {title, thought, action: callCode} = call.displayInfoFromArgs(args); |
| code = callCode; |
| if (title) { |
| yield { |
| type: ResponseType.TITLE, |
| title, |
| }; |
| } |
| |
| if (thought) { |
| yield { |
| type: ResponseType.THOUGHT, |
| thought, |
| }; |
| } |
| } |
| |
| const isOriginBlocked = (): boolean => { |
| const allowedOriginResult = this.#allowedOrigin?.(); |
| return Boolean(allowedOriginResult && 'blocked' in allowedOriginResult); |
| }; |
| |
| let result = await call.handler(args, options); |
| |
| // Check 1: After first handler execution. |
| // Navigation could have occurred during the async handler execution. |
| if (isOriginBlocked()) { |
| throw new CrossOriginError(); |
| } |
| |
| if ('requiresApproval' in result) { |
| if (code) { |
| yield { |
| type: ResponseType.ACTION, |
| code, |
| canceled: false, |
| }; |
| } |
| |
| const sideEffectConfirmationPromiseWithResolvers = this.confirmSideEffect<PermissionDecision>(); |
| |
| void sideEffectConfirmationPromiseWithResolvers.promise.then(decision => { |
| Host.userMetrics.actionTaken( |
| decision === PermissionDecision.REJECT ? Host.UserMetrics.Action.AiAssistanceSideEffectRejected : |
| Host.UserMetrics.Action.AiAssistanceSideEffectConfirmed, |
| ); |
| }); |
| |
| if (options?.signal?.aborted) { |
| sideEffectConfirmationPromiseWithResolvers.resolve(PermissionDecision.REJECT); |
| } |
| |
| const onAbort = (): void => { |
| sideEffectConfirmationPromiseWithResolvers.resolve(PermissionDecision.REJECT); |
| }; |
| |
| options?.signal?.addEventListener('abort', onAbort, {once: true}); |
| |
| yield { |
| type: ResponseType.SIDE_EFFECT, |
| confirm: sideEffectConfirmationPromiseWithResolvers.resolve, |
| description: result.description, |
| permissionPrompt: call.permissionPrompt, |
| permissionTitle: call.permissionTitle, |
| }; |
| |
| let decision = PermissionDecision.REJECT; |
| try { |
| decision = await sideEffectConfirmationPromiseWithResolvers.promise; |
| } finally { |
| options?.signal?.removeEventListener('abort', onAbort); |
| } |
| if (decision === PermissionDecision.REJECT) { |
| yield { |
| type: ResponseType.ACTION, |
| code, |
| output: 'Error: User denied code execution with side effects.', |
| canceled: true, |
| }; |
| debugLog(`[AiAgent] Tool '${name}' denied by user.`); |
| return { |
| result: 'Error: User denied code execution with side effects.', |
| }; |
| } |
| |
| // Re-check allowed origin after the approval await to prevent a TOCTOU (Time-of-Check |
| // to Time-of-Use) race condition where the page might have navigated cross-origin |
| // while the user was confirming the action. |
| // Check 2: After waiting for user approval. |
| if (isOriginBlocked()) { |
| throw new CrossOriginError(); |
| } |
| |
| result = await call.handler(args, { |
| ...options, |
| approved: true, |
| }); |
| |
| // Check 3: After second handler execution (approved run). |
| // Navigation could have occurred during the async execution of the approved action. |
| if (isOriginBlocked()) { |
| throw new CrossOriginError(); |
| } |
| } |
| |
| if ('result' in result) { |
| yield { |
| type: ResponseType.ACTION, |
| code, |
| output: typeof result.result === 'string' ? result.result : JSON.stringify(result.result), |
| widgets: result.widgets, |
| canceled: false, |
| toolName: name, |
| }; |
| } |
| |
| if ('error' in result) { |
| yield { |
| type: ResponseType.ACTION, |
| code, |
| output: result.error, |
| canceled: false, |
| toolName: name, |
| }; |
| } |
| |
| debugLog(`[AiAgent] Tool '${name}' result:`, result); |
| |
| if ('context' in result) { |
| return result; |
| } |
| |
| return result as {result: unknown, widgets?: AiWidget[]}; |
| } |
| |
| async * |
| #aidaFetch(request: Host.AidaClient.DoConversationRequest, options?: {signal?: AbortSignal}): |
| AsyncGenerator<AidaFetchResult, void, void> { |
| let aidaResponse: Host.AidaClient.DoConversationResponse|undefined = undefined; |
| let rpcId: Host.AidaClient.RpcGlobalId|undefined; |
| |
| for await (aidaResponse of this.#aidaClient.doConversation(request, options)) { |
| if (aidaResponse.functionCalls?.length) { |
| if (aidaResponse.functionCalls.length > 1) { |
| debugLog(`[AiAgent] Unexpected: received ${aidaResponse.functionCalls.length} function calls in response:`, |
| aidaResponse.functionCalls); |
| } |
| yield { |
| rpcId, |
| functionCall: aidaResponse.functionCalls[0], |
| completed: true, |
| text: aidaResponse.explanation, |
| }; |
| break; |
| } |
| |
| rpcId = aidaResponse.metadata.rpcGlobalId ?? rpcId; |
| yield { |
| rpcId, |
| text: aidaResponse.explanation, |
| completed: aidaResponse.completed, |
| }; |
| } |
| |
| if (isStructuredLogEnabled() && aidaResponse) { |
| this.#structuredLog.push({ |
| request: structuredClone(request), |
| aidaResponse, |
| }); |
| try { |
| localStorage.setItem('aiAssistanceStructuredLog', JSON.stringify(this.#structuredLog)); |
| } catch (err) { |
| console.warn('Failed to write to local storage "aiAssistanceStructuredLog":', err); |
| } |
| } |
| } |
| |
| #removeLastRunParts(): void { |
| this.#history.splice(this.#history.findLastIndex(item => { |
| return item.role === Host.AidaClient.Role.USER; |
| })); |
| } |
| |
| #createErrorResponse(error: ErrorType): ResponseData { |
| this.#removeLastRunParts(); |
| this.clearCache(); |
| if (error !== ErrorType.ABORT) { |
| Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceError); |
| } |
| |
| return { |
| type: ResponseType.ERROR, |
| error, |
| }; |
| } |
| } |
| |
| /** |
| * Parses `suggestions` as a JSON array and returns its non-empty string items, |
| * with whitespace collapsed and each item truncated to `MAX_SUGGESTION_LENGTH`. |
| * Returns `undefined` if the value is not an array or no items remain. |
| * Throws if `suggestions` is not valid JSON. |
| */ |
| export function sanitizeSuggestions(suggestions: string): [string, ...string[]]|undefined { |
| const parsed = JSON.parse(suggestions); |
| if (!Array.isArray(parsed)) { |
| return undefined; |
| } |
| const sanitized: string[] = []; |
| for (const item of parsed) { |
| if (typeof item !== 'string') { |
| continue; |
| } |
| // Collapse multiple whitespace/newlines into a single space. |
| const noExtraWhitespace = item.replace(/\s+/g, ' ').trim(); |
| if (noExtraWhitespace.length === 0) { |
| continue; |
| } |
| sanitized.push(noExtraWhitespace.substring(0, MAX_SUGGESTION_LENGTH)); |
| } |
| if (sanitized.length === 0) { |
| return undefined; |
| } |
| return sanitized as [string, ...string[]]; |
| } |
| |
| /** |
| * Maps AIDA-specific client error instances to user-facing ErrorType enums. |
| * This handles AIDA API failure modes such as quota exhaustion or blockages. |
| * Other application-level errors (like CROSS_ORIGIN or MAX_STEPS) are handled separately. |
| */ |
| export function aidaErrorToErrorType(err: unknown): ErrorType { |
| if (err instanceof Host.AidaClient.AidaAbortError) { |
| return ErrorType.ABORT; |
| } |
| if (err instanceof Host.AidaClient.AidaBlockError) { |
| return ErrorType.BLOCK; |
| } |
| if (err instanceof Host.AidaClient.AidaQuotaError) { |
| return ErrorType.QUOTA; |
| } |
| if (err instanceof Host.AidaClient.AidaPayloadTooLargeError) { |
| return ErrorType.PAYLOAD_TOO_LARGE; |
| } |
| return ErrorType.UNKNOWN; |
| } |
