diff --git a/.changeset/brave-llamas-impress.md b/.changeset/brave-llamas-impress.md new file mode 100644 index 0000000000..61d8f87ff4 --- /dev/null +++ b/.changeset/brave-llamas-impress.md @@ -0,0 +1,6 @@ +--- +'@firebase/ai': minor +'firebase': minor +--- + +Add `thoughtSummary()` convenience method to `EnhancedGenerateContentResponse`. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index e0eac35996..4a7afc765d 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -213,10 +213,10 @@ export { Date_2 as Date } // @public export interface EnhancedGenerateContentResponse extends GenerateContentResponse { - // (undocumented) functionCalls: () => FunctionCall[] | undefined; inlineDataParts: () => InlineDataPart[] | undefined; text: () => string; + thoughtSummary: () => string | undefined; } // @public @@ -249,6 +249,10 @@ export interface FileDataPart { inlineData?: never; // (undocumented) text?: never; + // (undocumented) + thought?: boolean; + // @internal (undocumented) + thoughtSignature?: never; } // @public @@ -303,6 +307,10 @@ export interface FunctionCallPart { inlineData?: never; // (undocumented) text?: never; + // (undocumented) + thought?: boolean; + // @internal (undocumented) + thoughtSignature?: never; } // @public @@ -335,6 +343,10 @@ export interface FunctionResponsePart { inlineData?: never; // (undocumented) text?: never; + // (undocumented) + thought?: boolean; + // @internal (undocumented) + thoughtSignature?: never; } // @public @@ -717,6 +729,10 @@ export interface InlineDataPart { inlineData: GenerativeContentBlob; // (undocumented) text?: never; + // (undocumented) + thought?: boolean; + // @internal (undocumented) + thoughtSignature?: never; videoMetadata?: VideoMetadata; } @@ -1048,10 +1064,15 @@ export interface TextPart { inlineData?: never; // (undocumented) text: string; + // (undocumented) + thought?: boolean; + // @internal (undocumented) + thoughtSignature?: string; } // @public export interface ThinkingConfig { + includeThoughts?: boolean; thinkingBudget?: number; } diff --git a/docs-devsite/ai.enhancedgeneratecontentresponse.md b/docs-devsite/ai.enhancedgeneratecontentresponse.md index 330dc10f32..9e947add0c 100644 --- a/docs-devsite/ai.enhancedgeneratecontentresponse.md +++ b/docs-devsite/ai.enhancedgeneratecontentresponse.md @@ -23,12 +23,15 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse | Property | Type | Description | | --- | --- | --- | -| [functionCalls](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () => [FunctionCall](./ai.functioncall.md#functioncall_interface)\[\] \| undefined | | -| [inlineDataParts](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () => [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)s from the [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | +| [functionCalls](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () => [FunctionCall](./ai.functioncall.md#functioncall_interface)\[\] \| undefined | Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | +| [inlineDataParts](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () => [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)\[\] \| undefined | Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [text](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () => string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. | +| [thoughtSummary](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsethoughtsummary) | () => string \| undefined | Aggregates and returns every [TextPart](./ai.textpart.md#textpart_interface) with their thought property set to true from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | ## EnhancedGenerateContentResponse.functionCalls +Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). + Signature: ```typescript @@ -37,7 +40,7 @@ functionCalls: () => FunctionCall[] | undefined; ## EnhancedGenerateContentResponse.inlineDataParts -Aggregates and returns all [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)s from the [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. +Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). Signature: @@ -54,3 +57,17 @@ Returns the text string from the response, if available. Throws if the prompt or ```typescript text: () => string; ``` + +## EnhancedGenerateContentResponse.thoughtSummary + +Aggregates and returns every [TextPart](./ai.textpart.md#textpart_interface) with their `thought` property set to `true` from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). + +Thought summaries provide a brief overview of the model's internal thinking process, offering insight into how it arrived at the final answer. This can be useful for debugging, understanding the model's reasoning, and verifying its accuracy. + +Thoughts will only be included if [ThinkingConfig.includeThoughts](./ai.thinkingconfig.md#thinkingconfigincludethoughts) is set to `true`. + +Signature: + +```typescript +thoughtSummary: () => string | undefined; +``` diff --git a/docs-devsite/ai.filedatapart.md b/docs-devsite/ai.filedatapart.md index 65cb9dc00e..2b5179319f 100644 --- a/docs-devsite/ai.filedatapart.md +++ b/docs-devsite/ai.filedatapart.md @@ -27,6 +27,7 @@ export interface FileDataPart | [functionResponse](./ai.filedatapart.md#filedatapartfunctionresponse) | never | | | [inlineData](./ai.filedatapart.md#filedatapartinlinedata) | never | | | [text](./ai.filedatapart.md#filedataparttext) | never | | +| [thought](./ai.filedatapart.md#filedatapartthought) | boolean | | ## FileDataPart.fileData @@ -67,3 +68,11 @@ inlineData?: never; ```typescript text?: never; ``` + +## FileDataPart.thought + +Signature: + +```typescript +thought?: boolean; +``` diff --git a/docs-devsite/ai.functioncallpart.md b/docs-devsite/ai.functioncallpart.md index b16e58f80a..3f07c5d0d7 100644 --- a/docs-devsite/ai.functioncallpart.md +++ b/docs-devsite/ai.functioncallpart.md @@ -26,6 +26,7 @@ export interface FunctionCallPart | [functionResponse](./ai.functioncallpart.md#functioncallpartfunctionresponse) | never | | | [inlineData](./ai.functioncallpart.md#functioncallpartinlinedata) | never | | | [text](./ai.functioncallpart.md#functioncallparttext) | never | | +| [thought](./ai.functioncallpart.md#functioncallpartthought) | boolean | | ## FunctionCallPart.functionCall @@ -58,3 +59,11 @@ inlineData?: never; ```typescript text?: never; ``` + +## FunctionCallPart.thought + +Signature: + +```typescript +thought?: boolean; +``` diff --git a/docs-devsite/ai.functionresponsepart.md b/docs-devsite/ai.functionresponsepart.md index 9c80258f43..4e8c9ea572 100644 --- a/docs-devsite/ai.functionresponsepart.md +++ b/docs-devsite/ai.functionresponsepart.md @@ -26,6 +26,7 @@ export interface FunctionResponsePart | [functionResponse](./ai.functionresponsepart.md#functionresponsepartfunctionresponse) | [FunctionResponse](./ai.functionresponse.md#functionresponse_interface) | | | [inlineData](./ai.functionresponsepart.md#functionresponsepartinlinedata) | never | | | [text](./ai.functionresponsepart.md#functionresponseparttext) | never | | +| [thought](./ai.functionresponsepart.md#functionresponsepartthought) | boolean | | ## FunctionResponsePart.functionCall @@ -58,3 +59,11 @@ inlineData?: never; ```typescript text?: never; ``` + +## FunctionResponsePart.thought + +Signature: + +```typescript +thought?: boolean; +``` diff --git a/docs-devsite/ai.inlinedatapart.md b/docs-devsite/ai.inlinedatapart.md index 0dd68edda6..c9ead9d061 100644 --- a/docs-devsite/ai.inlinedatapart.md +++ b/docs-devsite/ai.inlinedatapart.md @@ -26,6 +26,7 @@ export interface InlineDataPart | [functionResponse](./ai.inlinedatapart.md#inlinedatapartfunctionresponse) | never | | | [inlineData](./ai.inlinedatapart.md#inlinedatapartinlinedata) | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | | | [text](./ai.inlinedatapart.md#inlinedataparttext) | never | | +| [thought](./ai.inlinedatapart.md#inlinedatapartthought) | boolean | | | [videoMetadata](./ai.inlinedatapart.md#inlinedatapartvideometadata) | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Applicable if inlineData is a video. | ## InlineDataPart.functionCall @@ -60,6 +61,14 @@ inlineData: GenerativeContentBlob; text?: never; ``` +## InlineDataPart.thought + +Signature: + +```typescript +thought?: boolean; +``` + ## InlineDataPart.videoMetadata Applicable if `inlineData` is a video. diff --git a/docs-devsite/ai.textpart.md b/docs-devsite/ai.textpart.md index 2057d95d32..2466f9cca8 100644 --- a/docs-devsite/ai.textpart.md +++ b/docs-devsite/ai.textpart.md @@ -26,6 +26,7 @@ export interface TextPart | [functionResponse](./ai.textpart.md#textpartfunctionresponse) | never | | | [inlineData](./ai.textpart.md#textpartinlinedata) | never | | | [text](./ai.textpart.md#textparttext) | string | | +| [thought](./ai.textpart.md#textpartthought) | boolean | | ## TextPart.functionCall @@ -58,3 +59,11 @@ inlineData?: never; ```typescript text: string; ``` + +## TextPart.thought + +Signature: + +```typescript +thought?: boolean; +``` diff --git a/docs-devsite/ai.thinkingconfig.md b/docs-devsite/ai.thinkingconfig.md index ec348a2048..1ddc1626f4 100644 --- a/docs-devsite/ai.thinkingconfig.md +++ b/docs-devsite/ai.thinkingconfig.md @@ -24,8 +24,21 @@ export interface ThinkingConfig | Property | Type | Description | | --- | --- | --- | +| [includeThoughts](./ai.thinkingconfig.md#thinkingconfigincludethoughts) | boolean | Whether to include "thought summaries" in the model's response. | | [thinkingBudget](./ai.thinkingconfig.md#thinkingconfigthinkingbudget) | number | The thinking budget, in tokens.This parameter sets an upper limit on the number of tokens the model can use for its internal "thinking" process. A higher budget may result in higher quality responses for complex tasks but can also increase latency and cost.If you don't specify a budget, the model will determine the appropriate amount of thinking based on the complexity of the prompt.An error will be thrown if you set a thinking budget for a model that does not support this feature or if the specified budget is not within the model's supported range. | +## ThinkingConfig.includeThoughts + +Whether to include "thought summaries" in the model's response. + +Thought summaries provide a brief overview of the model's internal thinking process, offering insight into how it arrived at the final answer. This can be useful for debugging, understanding the model's reasoning, and verifying its accuracy. + +Signature: + +```typescript +includeThoughts?: boolean; +``` + ## ThinkingConfig.thinkingBudget The thinking budget, in tokens. diff --git a/packages/ai/src/methods/chat-session-helpers.test.ts b/packages/ai/src/methods/chat-session-helpers.test.ts index feab9fc3b0..e64f3e84e2 100644 --- a/packages/ai/src/methods/chat-session-helpers.test.ts +++ b/packages/ai/src/methods/chat-session-helpers.test.ts @@ -22,7 +22,11 @@ import { FirebaseError } from '@firebase/util'; describe('chat-session-helpers', () => { describe('validateChatHistory', () => { - const TCS: Array<{ history: Content[]; isValid: boolean }> = [ + const TCS: Array<{ + history: Content[]; + isValid: boolean; + errorShouldInclude?: string; + }> = [ { history: [{ role: 'user', parts: [{ text: 'hi' }] }], isValid: true @@ -99,19 +103,23 @@ describe('chat-session-helpers', () => { { //@ts-expect-error history: [{ role: 'user', parts: '' }], + errorShouldInclude: `array of Parts`, isValid: false }, { //@ts-expect-error history: [{ role: 'user' }], + errorShouldInclude: `array of Parts`, isValid: false }, { history: [{ role: 'user', parts: [] }], + errorShouldInclude: `at least one part`, isValid: false }, { history: [{ role: 'model', parts: [{ text: 'hi' }] }], + errorShouldInclude: `model`, isValid: false }, { @@ -125,6 +133,7 @@ describe('chat-session-helpers', () => { ] } ], + errorShouldInclude: `function`, isValid: false }, { @@ -132,6 +141,7 @@ describe('chat-session-helpers', () => { { role: 'user', parts: [{ text: 'hi' }] }, { role: 'user', parts: [{ text: 'hi' }] } ], + errorShouldInclude: `can't follow 'user'`, isValid: false }, { @@ -140,6 +150,45 @@ describe('chat-session-helpers', () => { { role: 'model', parts: [{ text: 'hi' }] }, { role: 'model', parts: [{ text: 'hi' }] } ], + errorShouldInclude: `can't follow 'model'`, + isValid: false + }, + { + history: [ + { role: 'user', parts: [{ text: 'hi' }] }, + { + role: 'model', + parts: [ + { text: 'hi' }, + { + text: 'thought about hi', + thought: true, + thoughtSignature: 'thought signature' + } + ] + } + ], + isValid: true + }, + { + history: [ + { + role: 'user', + parts: [{ text: 'hi', thought: true, thoughtSignature: 'sig' }] + }, + { + role: 'model', + parts: [ + { text: 'hi' }, + { + text: 'thought about hi', + thought: true, + thoughtSignature: 'thought signature' + } + ] + } + ], + errorShouldInclude: 'thought', isValid: false } ]; @@ -149,7 +198,14 @@ describe('chat-session-helpers', () => { if (tc.isValid) { expect(fn).to.not.throw(); } else { - expect(fn).to.throw(FirebaseError); + try { + fn(); + } catch (e) { + expect(e).to.be.instanceOf(FirebaseError); + if (e instanceof FirebaseError && tc.errorShouldInclude) { + expect(e.message).to.include(tc.errorShouldInclude); + } + } } }); }); diff --git a/packages/ai/src/methods/chat-session-helpers.ts b/packages/ai/src/methods/chat-session-helpers.ts index 1bb0e2798f..709f616f3c 100644 --- a/packages/ai/src/methods/chat-session-helpers.ts +++ b/packages/ai/src/methods/chat-session-helpers.ts @@ -24,13 +24,15 @@ const VALID_PART_FIELDS: Array = [ 'text', 'inlineData', 'functionCall', - 'functionResponse' + 'functionResponse', + 'thought', + 'thoughtSignature' ]; const VALID_PARTS_PER_ROLE: { [key in Role]: Array } = { user: ['text', 'inlineData'], function: ['functionResponse'], - model: ['text', 'functionCall'], + model: ['text', 'functionCall', 'thought', 'thoughtSignature'], // System instructions shouldn't be in history anyway. system: ['text'] }; @@ -65,7 +67,7 @@ export function validateChatHistory(history: Content[]): void { if (!Array.isArray(parts)) { throw new AIError( AIErrorCode.INVALID_CONTENT, - `Content should have 'parts' but property with an array of Parts` + `Content should have 'parts' property with an array of Parts` ); } @@ -80,7 +82,9 @@ export function validateChatHistory(history: Content[]): void { text: 0, inlineData: 0, functionCall: 0, - functionResponse: 0 + functionResponse: 0, + thought: 0, + thoughtSignature: 0 }; for (const part of parts) { diff --git a/packages/ai/src/methods/chat-session.test.ts b/packages/ai/src/methods/chat-session.test.ts index f523672f5e..e92aa057af 100644 --- a/packages/ai/src/methods/chat-session.test.ts +++ b/packages/ai/src/methods/chat-session.test.ts @@ -20,7 +20,7 @@ import { match, restore, stub, useFakeTimers } from 'sinon'; import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; import * as generateContentMethods from './generate-content'; -import { GenerateContentStreamResult, InferenceMode } from '../types'; +import { Content, GenerateContentStreamResult, InferenceMode } from '../types'; import { ChatSession } from './chat-session'; import { ApiSettings } from '../types/internal'; import { VertexAIBackend } from '../backend'; @@ -65,6 +65,53 @@ describe('ChatSession', () => { match.any ); }); + it('adds message and response to history', async () => { + const fakeContent: Content = { + role: 'model', + parts: [ + { text: 'hi' }, + { + text: 'thought about hi', + thoughtSignature: 'thought signature' + } + ] + }; + const fakeResponse = { + candidates: [ + { + index: 1, + content: fakeContent + } + ] + }; + const generateContentStub = stub( + generateContentMethods, + 'generateContent' + ).resolves({ + // @ts-ignore + response: fakeResponse + }); + const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const result = await chatSession.sendMessage('hello'); + // @ts-ignore + expect(result.response).to.equal(fakeResponse); + // Test: stores history correctly? + const history = await chatSession.getHistory(); + expect(history[0].role).to.equal('user'); + expect(history[0].parts[0].text).to.equal('hello'); + expect(history[1]).to.deep.equal(fakeResponse.candidates[0].content); + // Test: sends history correctly? + await chatSession.sendMessage('hello 2'); + expect(generateContentStub.args[1][2].contents[0].parts[0].text).to.equal( + 'hello' + ); + expect(generateContentStub.args[1][2].contents[1]).to.deep.equal( + fakeResponse.candidates[0].content + ); + expect(generateContentStub.args[1][2].contents[2].parts[0].text).to.equal( + 'hello 2' + ); + }); }); describe('sendMessageStream()', () => { it('generateContentStream errors should be catchable', async () => { diff --git a/packages/ai/src/requests/response-helpers.test.ts b/packages/ai/src/requests/response-helpers.test.ts index 97dd2f9fe3..8583ca9a73 100644 --- a/packages/ai/src/requests/response-helpers.test.ts +++ b/packages/ai/src/requests/response-helpers.test.ts @@ -48,6 +48,21 @@ const fakeResponseText: GenerateContentResponse = { ] }; +const fakeResponseThoughts: GenerateContentResponse = { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [ + { text: 'Some text' }, + { text: 'and some thoughts', thought: true } + ] + } + } + ] +}; + const functionCallPart1 = { functionCall: { name: 'find_theaters', @@ -188,6 +203,7 @@ describe('response-helpers methods', () => { expect(enhancedResponse.text()).to.equal('Some text and some more text'); expect(enhancedResponse.functionCalls()).to.be.undefined; expect(enhancedResponse.inlineDataParts()).to.be.undefined; + expect(enhancedResponse.thoughtSummary()).to.be.undefined; }); it('good response functionCall', async () => { const enhancedResponse = addHelpers(fakeResponseFunctionCall); @@ -196,6 +212,7 @@ describe('response-helpers methods', () => { functionCallPart1.functionCall ]); expect(enhancedResponse.inlineDataParts()).to.be.undefined; + expect(enhancedResponse.thoughtSummary()).to.be.undefined; }); it('good response functionCalls', async () => { const enhancedResponse = addHelpers(fakeResponseFunctionCalls); @@ -205,6 +222,7 @@ describe('response-helpers methods', () => { functionCallPart2.functionCall ]); expect(enhancedResponse.inlineDataParts()).to.be.undefined; + expect(enhancedResponse.thoughtSummary()).to.be.undefined; }); it('good response text/functionCall', async () => { const enhancedResponse = addHelpers(fakeResponseMixed1); @@ -213,6 +231,7 @@ describe('response-helpers methods', () => { ]); expect(enhancedResponse.text()).to.equal('some text'); expect(enhancedResponse.inlineDataParts()).to.be.undefined; + expect(enhancedResponse.thoughtSummary()).to.be.undefined; }); it('good response functionCall/text', async () => { const enhancedResponse = addHelpers(fakeResponseMixed2); @@ -221,6 +240,7 @@ describe('response-helpers methods', () => { ]); expect(enhancedResponse.text()).to.equal('some text'); expect(enhancedResponse.inlineDataParts()).to.be.undefined; + expect(enhancedResponse.thoughtSummary()).to.be.undefined; }); it('good response text/functionCall/text', async () => { const enhancedResponse = addHelpers(fakeResponseMixed3); @@ -228,17 +248,20 @@ describe('response-helpers methods', () => { functionCallPart1.functionCall ]); expect(enhancedResponse.text()).to.equal('some text and more text'); + expect(enhancedResponse.thoughtSummary()).to.be.undefined; expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('bad response safety', async () => { const enhancedResponse = addHelpers(badFakeResponse); expect(enhancedResponse.text).to.throw('SAFETY'); + expect(enhancedResponse.thoughtSummary).to.throw('SAFETY'); expect(enhancedResponse.functionCalls).to.throw('SAFETY'); expect(enhancedResponse.inlineDataParts).to.throw('SAFETY'); }); it('good response inlineData', async () => { const enhancedResponse = addHelpers(fakeResponseInlineData); expect(enhancedResponse.text()).to.equal(''); + expect(enhancedResponse.thoughtSummary()).to.be.undefined; expect(enhancedResponse.functionCalls()).to.be.undefined; expect(enhancedResponse.inlineDataParts()).to.deep.equal([ inlineDataPart1, @@ -248,11 +271,19 @@ describe('response-helpers methods', () => { it('good response text/inlineData', async () => { const enhancedResponse = addHelpers(fakeResponseTextAndInlineData); expect(enhancedResponse.text()).to.equal('Describe this:'); + expect(enhancedResponse.thoughtSummary()).to.be.undefined; expect(enhancedResponse.functionCalls()).to.be.undefined; expect(enhancedResponse.inlineDataParts()).to.deep.equal([ inlineDataPart1 ]); }); + it('good response text/thought', async () => { + const enhancedResponse = addHelpers(fakeResponseThoughts); + expect(enhancedResponse.text()).to.equal('Some text'); + expect(enhancedResponse.thoughtSummary()).to.equal('and some thoughts'); + expect(enhancedResponse.functionCalls()).to.be.undefined; + expect(enhancedResponse.inlineDataParts()).to.be.undefined; + }); }); describe('getBlockString', () => { it('has no promptFeedback or bad finishReason', async () => { diff --git a/packages/ai/src/requests/response-helpers.ts b/packages/ai/src/requests/response-helpers.ts index 2505b5c927..16d5561348 100644 --- a/packages/ai/src/requests/response-helpers.ts +++ b/packages/ai/src/requests/response-helpers.ts @@ -24,12 +24,43 @@ import { ImagenGCSImage, ImagenInlineImage, AIErrorCode, - InlineDataPart + InlineDataPart, + Part } from '../types'; import { AIError } from '../errors'; import { logger } from '../logger'; import { ImagenResponseInternal } from '../types/internal'; +/** + * Check that at least one candidate exists and does not have a bad + * finish reason. Warns if multiple candidates exist. + */ +function hasValidCandidates(response: GenerateContentResponse): boolean { + if (response.candidates && response.candidates.length > 0) { + if (response.candidates.length > 1) { + logger.warn( + `This response had ${response.candidates.length} ` + + `candidates. Returning text from the first candidate only. ` + + `Access response.candidates directly to use the other candidates.` + ); + } + if (hadBadFinishReason(response.candidates[0])) { + throw new AIError( + AIErrorCode.RESPONSE_ERROR, + `Response error: ${formatBlockErrorMessage( + response + )}. Response body stored in error.response`, + { + response + } + ); + } + return true; + } else { + return false; + } +} + /** * Creates an EnhancedGenerateContentResponse object that has helper functions and * other modifications that improve usability. @@ -59,26 +90,8 @@ export function addHelpers( response: GenerateContentResponse ): EnhancedGenerateContentResponse { (response as EnhancedGenerateContentResponse).text = () => { - if (response.candidates && response.candidates.length > 0) { - if (response.candidates.length > 1) { - logger.warn( - `This response had ${response.candidates.length} ` + - `candidates. Returning text from the first candidate only. ` + - `Access response.candidates directly to use the other candidates.` - ); - } - if (hadBadFinishReason(response.candidates[0])) { - throw new AIError( - AIErrorCode.RESPONSE_ERROR, - `Response error: ${formatBlockErrorMessage( - response - )}. Response body stored in error.response`, - { - response - } - ); - } - return getText(response); + if (hasValidCandidates(response)) { + return getText(response, part => !part.thought); } else if (response.promptFeedback) { throw new AIError( AIErrorCode.RESPONSE_ERROR, @@ -90,28 +103,25 @@ export function addHelpers( } return ''; }; + (response as EnhancedGenerateContentResponse).thoughtSummary = () => { + if (hasValidCandidates(response)) { + const result = getText(response, part => !!part.thought); + return result === '' ? undefined : result; + } else if (response.promptFeedback) { + throw new AIError( + AIErrorCode.RESPONSE_ERROR, + `Thought summary not available. ${formatBlockErrorMessage(response)}`, + { + response + } + ); + } + return undefined; + }; (response as EnhancedGenerateContentResponse).inlineDataParts = (): | InlineDataPart[] | undefined => { - if (response.candidates && response.candidates.length > 0) { - if (response.candidates.length > 1) { - logger.warn( - `This response had ${response.candidates.length} ` + - `candidates. Returning data from the first candidate only. ` + - `Access response.candidates directly to use the other candidates.` - ); - } - if (hadBadFinishReason(response.candidates[0])) { - throw new AIError( - AIErrorCode.RESPONSE_ERROR, - `Response error: ${formatBlockErrorMessage( - response - )}. Response body stored in error.response`, - { - response - } - ); - } + if (hasValidCandidates(response)) { return getInlineDataParts(response); } else if (response.promptFeedback) { throw new AIError( @@ -125,25 +135,7 @@ export function addHelpers( return undefined; }; (response as EnhancedGenerateContentResponse).functionCalls = () => { - if (response.candidates && response.candidates.length > 0) { - if (response.candidates.length > 1) { - logger.warn( - `This response had ${response.candidates.length} ` + - `candidates. Returning function calls from the first candidate only. ` + - `Access response.candidates directly to use the other candidates.` - ); - } - if (hadBadFinishReason(response.candidates[0])) { - throw new AIError( - AIErrorCode.RESPONSE_ERROR, - `Response error: ${formatBlockErrorMessage( - response - )}. Response body stored in error.response`, - { - response - } - ); - } + if (hasValidCandidates(response)) { return getFunctionCalls(response); } else if (response.promptFeedback) { throw new AIError( @@ -160,13 +152,20 @@ export function addHelpers( } /** - * Returns all text found in all parts of first candidate. + * Returns all text from the first candidate's parts, filtering by whether + * `partFilter()` returns true. + * + * @param response - The `GenerateContentResponse` from which to extract text. + * @param partFilter - Only return `Part`s for which this returns true */ -export function getText(response: GenerateContentResponse): string { +export function getText( + response: GenerateContentResponse, + partFilter: (part: Part) => boolean +): string { const textStrings = []; if (response.candidates?.[0].content?.parts) { for (const part of response.candidates?.[0].content?.parts) { - if (part.text) { + if (part.text && partFilter(part)) { textStrings.push(part.text); } } @@ -179,7 +178,7 @@ export function getText(response: GenerateContentResponse): string { } /** - * Returns {@link FunctionCall}s associated with first candidate. + * Returns every {@link FunctionCall} associated with first candidate. */ export function getFunctionCalls( response: GenerateContentResponse @@ -200,7 +199,7 @@ export function getFunctionCalls( } /** - * Returns {@link InlineDataPart}s in the first candidate if present. + * Returns every {@link InlineDataPart} in the first candidate if present. * * @internal */ diff --git a/packages/ai/src/types/content.ts b/packages/ai/src/types/content.ts index ad2906671e..a08af95086 100644 --- a/packages/ai/src/types/content.ts +++ b/packages/ai/src/types/content.ts @@ -47,6 +47,11 @@ export interface TextPart { inlineData?: never; functionCall?: never; functionResponse?: never; + thought?: boolean; + /** + * @internal + */ + thoughtSignature?: string; } /** @@ -62,6 +67,11 @@ export interface InlineDataPart { * Applicable if `inlineData` is a video. */ videoMetadata?: VideoMetadata; + thought?: boolean; + /** + * @internal + */ + thoughtSignature?: never; } /** @@ -90,6 +100,11 @@ export interface FunctionCallPart { inlineData?: never; functionCall: FunctionCall; functionResponse?: never; + thought?: boolean; + /** + * @internal + */ + thoughtSignature?: never; } /** @@ -101,6 +116,11 @@ export interface FunctionResponsePart { inlineData?: never; functionCall?: never; functionResponse: FunctionResponse; + thought?: boolean; + /** + * @internal + */ + thoughtSignature?: never; } /** @@ -113,6 +133,11 @@ export interface FileDataPart { functionCall?: never; functionResponse?: never; fileData: FileData; + thought?: boolean; + /** + * @internal + */ + thoughtSignature?: never; } /** diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index ce18710192..93921c6f14 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -333,4 +333,14 @@ export interface ThinkingConfig { * feature or if the specified budget is not within the model's supported range. */ thinkingBudget?: number; + + /** + * Whether to include "thought summaries" in the model's response. + * + * @remarks + * Thought summaries provide a brief overview of the model's internal thinking process, + * offering insight into how it arrived at the final answer. This can be useful for + * debugging, understanding the model's reasoning, and verifying its accuracy. + */ + includeThoughts?: boolean; } diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index 323699e646..d9b76155a3 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -60,15 +60,34 @@ export interface EnhancedGenerateContentResponse */ text: () => string; /** - * Aggregates and returns all {@link InlineDataPart}s from the {@link GenerateContentResponse}'s - * first candidate. - * - * @returns An array of {@link InlineDataPart}s containing data from the response, if available. + * Aggregates and returns every {@link InlineDataPart} from the first candidate of + * {@link GenerateContentResponse}. * * @throws If the prompt or candidate was blocked. */ inlineDataParts: () => InlineDataPart[] | undefined; + /** + * Aggregates and returns every {@link FunctionCall} from the first candidate of + * {@link GenerateContentResponse}. + * + * @throws If the prompt or candidate was blocked. + */ functionCalls: () => FunctionCall[] | undefined; + /** + * Aggregates and returns every {@link TextPart} with their `thought` property set + * to `true` from the first candidate of {@link GenerateContentResponse}. + * + * @throws If the prompt or candidate was blocked. + * + * @remarks + * Thought summaries provide a brief overview of the model's internal thinking process, + * offering insight into how it arrived at the final answer. This can be useful for + * debugging, understanding the model's reasoning, and verifying its accuracy. + * + * Thoughts will only be included if {@link ThinkingConfig.includeThoughts} is + * set to `true`. + */ + thoughtSummary: () => string | undefined; } /**