From 95c3556ccb2481091219ecdd2225b8ca29909620 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Sat, 8 Feb 2025 18:06:48 +0100 Subject: [PATCH 01/11] perf(embedding): always request embedding creation as base64 Requesting base64 encoded embeddings returns smaller body sizes, on average ~60% smaller than float32 encoded. In other words, the size of the response body containing embeddings in float32 is ~2.3x bigger than base64 encoded embedding. We always request embedding creating encoded as base64, and then decoded them to float32 based on the user's provided encoding_format parameter. Closes #1310 --- src/resources/embeddings.ts | 41 ++++++++++++++++++++++++-- tests/api-resources/embeddings.test.ts | 12 ++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index d01ffc807a..84c4e87939 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -9,9 +9,46 @@ export class Embeddings extends APIResource { */ create( body: EmbeddingCreateParams, - options?: Core.RequestOptions, + options?: Core.RequestOptions, ): Core.APIPromise { - return this._client.post('/embeddings', { body, ...options }); + const userInitialEncodingFormat = body.encoding_format; + Core.debug('request', 'Sending request with arguments:', { body, ...options }); + + const base64Response = this._client.post('/embeddings', { + body: { + ...body, + // Force base64 encoding for vector embeddings creation + // See https://github.com/openai/openai-node/issues/1310 + encoding_format: 'base64', + }, + ...options, + }); + + if (userInitialEncodingFormat === 'base64') { + // if the user requested base64 encoding_format, return the response as-is + return base64Response; + } else { + // we decode the base64 embeddings to float32 array if: + // 1- the user requested 'float' encoding_format, + // 2- the user did not specify an encoding_format (which defaults to 'float') in order to keep backwards compatibility + Core.debug('response', `User requested encoding_format=${userInitialEncodingFormat || 'default'}`); + Core.debug('response', 'Decoding base64 embeddings to float32 array'); + + return base64Response._thenUnwrap((response) => { + if (response && response.data) { + response.data.forEach((embeddingBase64Obj) => { + console.log(embeddingBase64Obj); + const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; + embeddingBase64Obj.embedding = Array.from( + new Float32Array(Buffer.from(embeddingBase64Str, 'base64').buffer), + ); + }); + Core.debug('response', 'Decoded embeddings:', response.data); + } + + return response; + }); + } } } diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index 46dd1b2a3f..a9a62c0f02 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -32,4 +32,16 @@ describe('resource embeddings', () => { user: 'user-1234', }); }); + + test('create: encoding_format=default should create float32 embeddings', async () => { + const responsePromise = client.embeddings.create({ + input: 'The quick brown fox jumped over the lazy dog', + model: 'text-embedding-3-small', + }); + const response = await responsePromise; + console.log(response.data?.at(0)?.embedding); + + expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); + expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); + }); }); From 992d731ff3d590aa1d6bcbcf00514ad87145267a Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Mon, 24 Feb 2025 21:49:17 +0100 Subject: [PATCH 02/11] chore: move toFloat32Array to core.ts --- src/core.ts | 21 +++++++++++++++++++++ src/resources/embeddings.ts | 5 +---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/core.ts b/src/core.ts index 0dedc53eb5..a3f6649061 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1287,6 +1287,27 @@ export const toBase64 = (str: string | null | undefined): string => { throw new OpenAIError('Cannot generate b64 string; Expected `Buffer` or `btoa` to be defined'); }; +/** + * Converts a Base64 encoded string to a Float32Array. + * @param base64Str - The Base64 encoded string. + * @returns An Array of numbers interpreted as Float32 values. + */ +export const toFloat32Array = (base64Str: string): Array => { + if (typeof Buffer !== 'undefined') { + // for Node.js environment + return Array.from(new Float32Array(Buffer.from(base64Str, 'base64').buffer)); + } else { + // for legacy web platform APIs + const binaryStr = atob(base64Str); + const len = binaryStr.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + return Array.from(new Float32Array(bytes.buffer)); + } +}; + export function isObj(obj: unknown): obj is Record { return obj != null && typeof obj === 'object' && !Array.isArray(obj); } diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index 84c4e87939..275454f6d1 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -37,11 +37,8 @@ export class Embeddings extends APIResource { return base64Response._thenUnwrap((response) => { if (response && response.data) { response.data.forEach((embeddingBase64Obj) => { - console.log(embeddingBase64Obj); const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; - embeddingBase64Obj.embedding = Array.from( - new Float32Array(Buffer.from(embeddingBase64Str, 'base64').buffer), - ); + embeddingBase64Obj.embedding = Core.toFloat32Array(embeddingBase64Str); }); Core.debug('response', 'Decoded embeddings:', response.data); } From d8dde73dc1f4e83153f531c156ed4e37d26f45fc Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Mon, 24 Feb 2025 22:06:19 +0100 Subject: [PATCH 03/11] chore: default to base64 if user didn't provide encoding_format Closes #1310 --- src/resources/embeddings.ts | 61 +++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index 275454f6d1..abb8586525 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -11,41 +11,50 @@ export class Embeddings extends APIResource { body: EmbeddingCreateParams, options?: Core.RequestOptions, ): Core.APIPromise { - const userInitialEncodingFormat = body.encoding_format; Core.debug('request', 'Sending request with arguments:', { body, ...options }); - const base64Response = this._client.post('/embeddings', { + const hasUserProvidedEncodingFormat = body.encoding_format !== undefined; + let encoding_format: 'float' | 'base64' = 'float'; // current API defaults to float + + if (hasUserProvidedEncodingFormat === false) { + // No encoding_format specified, defaulting to base64 for performance reasons + // See https://github.com/openai/openai-node/pull/1312 + encoding_format = 'base64'; + } else { + Core.debug('Request', 'User defined encoding_format:', body.encoding_format); + } + + const response = this._client.post('/embeddings', { body: { ...body, - // Force base64 encoding for vector embeddings creation - // See https://github.com/openai/openai-node/issues/1310 - encoding_format: 'base64', + encoding_format, }, ...options, }); - if (userInitialEncodingFormat === 'base64') { - // if the user requested base64 encoding_format, return the response as-is - return base64Response; - } else { - // we decode the base64 embeddings to float32 array if: - // 1- the user requested 'float' encoding_format, - // 2- the user did not specify an encoding_format (which defaults to 'float') in order to keep backwards compatibility - Core.debug('response', `User requested encoding_format=${userInitialEncodingFormat || 'default'}`); - Core.debug('response', 'Decoding base64 embeddings to float32 array'); - - return base64Response._thenUnwrap((response) => { - if (response && response.data) { - response.data.forEach((embeddingBase64Obj) => { - const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; - embeddingBase64Obj.embedding = Core.toFloat32Array(embeddingBase64Str); - }); - Core.debug('response', 'Decoded embeddings:', response.data); - } - - return response; - }); + // if the user specified an encoding_format, return the response as-is + if (hasUserProvidedEncodingFormat) { + return response; } + + // in this stage, we are sure the user did not specify an encoding_format + // and we defaulted to base64 for performance reasons + // we are sure then that the response is base64 encoded, let's decode it + // the returned result will be a float32 array since this is OpenAI API's default encoding + Core.debug('response', `User requested encoding_format=${encoding_format || 'default'}`); + Core.debug('response', 'Decoding base64 embeddings to float32 array'); + + return response._thenUnwrap((response) => { + if (response && response.data) { + response.data.forEach((embeddingBase64Obj) => { + const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; + embeddingBase64Obj.embedding = Core.toFloat32Array(embeddingBase64Str); + }); + Core.debug('response', 'Decoded embeddings:', response.data); + } + + return response; + }); } } From bb0f8da5eb689f86081d3999c7008151798ea42a Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Mon, 24 Feb 2025 23:06:23 +0100 Subject: [PATCH 04/11] chore: update tests --- src/resources/embeddings.ts | 19 ++++++++++--------- tests/api-resources/embeddings.test.ts | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index abb8586525..54a0e60a2f 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -13,28 +13,29 @@ export class Embeddings extends APIResource { ): Core.APIPromise { Core.debug('request', 'Sending request with arguments:', { body, ...options }); - const hasUserProvidedEncodingFormat = body.encoding_format !== undefined; - let encoding_format: 'float' | 'base64' = 'float'; // current API defaults to float + const hasUserProvidedEncodingFormat = !!body.encoding_format; + let encoding_format: EmbeddingCreateParams['encoding_format'] = + hasUserProvidedEncodingFormat ? body.encoding_format : 'base64'; - if (hasUserProvidedEncodingFormat === false) { + if (hasUserProvidedEncodingFormat) { + Core.debug('Request', 'User defined encoding_format:', body.encoding_format); + } else { // No encoding_format specified, defaulting to base64 for performance reasons // See https://github.com/openai/openai-node/pull/1312 encoding_format = 'base64'; - } else { - Core.debug('Request', 'User defined encoding_format:', body.encoding_format); } - const response = this._client.post('/embeddings', { + const response = this._client.post('/embeddings', { body: { ...body, - encoding_format, + encoding_format: encoding_format as EmbeddingCreateParams['encoding_format'], }, ...options, }); // if the user specified an encoding_format, return the response as-is if (hasUserProvidedEncodingFormat) { - return response; + return response as Core.APIPromise; } // in this stage, we are sure the user did not specify an encoding_format @@ -44,7 +45,7 @@ export class Embeddings extends APIResource { Core.debug('response', `User requested encoding_format=${encoding_format || 'default'}`); Core.debug('response', 'Decoding base64 embeddings to float32 array'); - return response._thenUnwrap((response) => { + return (response as Core.APIPromise)._thenUnwrap((response) => { if (response && response.data) { response.data.forEach((embeddingBase64Obj) => { const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index a9a62c0f02..c2f1b91445 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -33,13 +33,35 @@ describe('resource embeddings', () => { }); }); + test.only('create: encoding_format=float should create float32 embeddings', async () => { + const responsePromise = client.embeddings.create({ + input: 'The quick brown fox jumped over the lazy dog', + model: 'text-embedding-3-small', + }); + const response = await responsePromise; + + expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); + expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); + }); + + test('create: encoding_format=base64 should create float32 embeddings', async () => { + const responsePromise = client.embeddings.create({ + input: 'The quick brown fox jumped over the lazy dog', + model: 'text-embedding-3-small', + encoding_format: 'base64', + }); + const response = await responsePromise; + + expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); + expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); + }); + test('create: encoding_format=default should create float32 embeddings', async () => { const responsePromise = client.embeddings.create({ input: 'The quick brown fox jumped over the lazy dog', model: 'text-embedding-3-small', }); const response = await responsePromise; - console.log(response.data?.at(0)?.embedding); expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); From f17bc7393f076e74287f14aa2e73a90b2d0f34f4 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:47:42 +0100 Subject: [PATCH 05/11] Update tests/api-resources/embeddings.test.ts Co-authored-by: Robert Craigie --- tests/api-resources/embeddings.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index c2f1b91445..08d421a627 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -33,7 +33,7 @@ describe('resource embeddings', () => { }); }); - test.only('create: encoding_format=float should create float32 embeddings', async () => { + test('create: encoding_format=float should create float32 embeddings', async () => { const responsePromise = client.embeddings.create({ input: 'The quick brown fox jumped over the lazy dog', model: 'text-embedding-3-small', From c3bda2e4af05049cdcdfa2f4fbdf64265cec1c47 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:47:54 +0100 Subject: [PATCH 06/11] Update tests/api-resources/embeddings.test.ts Co-authored-by: Robert Craigie --- tests/api-resources/embeddings.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index 08d421a627..c23fc65434 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -45,12 +45,11 @@ describe('resource embeddings', () => { }); test('create: encoding_format=base64 should create float32 embeddings', async () => { - const responsePromise = client.embeddings.create({ + const response = await client.embeddings.create({ input: 'The quick brown fox jumped over the lazy dog', model: 'text-embedding-3-small', encoding_format: 'base64', }); - const response = await responsePromise; expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); From de662fad80545dabefbd27b515ef128710760b2d Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:48:03 +0100 Subject: [PATCH 07/11] Update tests/api-resources/embeddings.test.ts Co-authored-by: Robert Craigie --- tests/api-resources/embeddings.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index c23fc65434..45b5e8ecb1 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -34,11 +34,10 @@ describe('resource embeddings', () => { }); test('create: encoding_format=float should create float32 embeddings', async () => { - const responsePromise = client.embeddings.create({ + const response = await client.embeddings.create({ input: 'The quick brown fox jumped over the lazy dog', model: 'text-embedding-3-small', }); - const response = await responsePromise; expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); From ec27e056d755e6c7ee040c664df4b905dcb23c60 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:48:17 +0100 Subject: [PATCH 08/11] Update src/resources/embeddings.ts Co-authored-by: Robert Craigie --- src/resources/embeddings.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index 54a0e60a2f..fee07ac6bb 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -42,7 +42,6 @@ export class Embeddings extends APIResource { // and we defaulted to base64 for performance reasons // we are sure then that the response is base64 encoded, let's decode it // the returned result will be a float32 array since this is OpenAI API's default encoding - Core.debug('response', `User requested encoding_format=${encoding_format || 'default'}`); Core.debug('response', 'Decoding base64 embeddings to float32 array'); return (response as Core.APIPromise)._thenUnwrap((response) => { From ca9a5cbefc88ed76dc349d17827c6fb45a0f0a52 Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:51:09 +0100 Subject: [PATCH 09/11] Update tests/api-resources/embeddings.test.ts Co-authored-by: Robert Craigie --- tests/api-resources/embeddings.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/api-resources/embeddings.test.ts b/tests/api-resources/embeddings.test.ts index 45b5e8ecb1..e226ade9eb 100644 --- a/tests/api-resources/embeddings.test.ts +++ b/tests/api-resources/embeddings.test.ts @@ -55,11 +55,10 @@ describe('resource embeddings', () => { }); test('create: encoding_format=default should create float32 embeddings', async () => { - const responsePromise = client.embeddings.create({ + const response = await client.embeddings.create({ input: 'The quick brown fox jumped over the lazy dog', model: 'text-embedding-3-small', }); - const response = await responsePromise; expect(response.data?.at(0)?.embedding).toBeInstanceOf(Array); expect(Number.isFinite(response.data?.at(0)?.embedding.at(0))).toBe(true); From d2bc20bdf2daf89259324d605ee0f12e284e162b Mon Sep 17 00:00:00 2001 From: Wassim Chegham Date: Tue, 18 Mar 2025 14:55:27 +0100 Subject: [PATCH 10/11] fix: refactor encoding_format handling for clarity --- src/resources/embeddings.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index fee07ac6bb..adb241ea1b 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -14,18 +14,16 @@ export class Embeddings extends APIResource { Core.debug('request', 'Sending request with arguments:', { body, ...options }); const hasUserProvidedEncodingFormat = !!body.encoding_format; + // No encoding_format specified, defaulting to base64 for performance reasons + // See https://github.com/openai/openai-node/pull/1312 let encoding_format: EmbeddingCreateParams['encoding_format'] = hasUserProvidedEncodingFormat ? body.encoding_format : 'base64'; if (hasUserProvidedEncodingFormat) { Core.debug('Request', 'User defined encoding_format:', body.encoding_format); - } else { - // No encoding_format specified, defaulting to base64 for performance reasons - // See https://github.com/openai/openai-node/pull/1312 - encoding_format = 'base64'; } - const response = this._client.post('/embeddings', { + const response: Core.APIPromise = this._client.post('/embeddings', { body: { ...body, encoding_format: encoding_format as EmbeddingCreateParams['encoding_format'], @@ -35,7 +33,7 @@ export class Embeddings extends APIResource { // if the user specified an encoding_format, return the response as-is if (hasUserProvidedEncodingFormat) { - return response as Core.APIPromise; + return response; } // in this stage, we are sure the user did not specify an encoding_format From d20005bb6f94cd2f5a0470cd3e02196e495876b0 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Fri, 28 Mar 2025 20:43:42 +0000 Subject: [PATCH 11/11] remove some debug logs --- src/resources/embeddings.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index adb241ea1b..a4be9ca3c7 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -11,8 +11,6 @@ export class Embeddings extends APIResource { body: EmbeddingCreateParams, options?: Core.RequestOptions, ): Core.APIPromise { - Core.debug('request', 'Sending request with arguments:', { body, ...options }); - const hasUserProvidedEncodingFormat = !!body.encoding_format; // No encoding_format specified, defaulting to base64 for performance reasons // See https://github.com/openai/openai-node/pull/1312 @@ -48,7 +46,6 @@ export class Embeddings extends APIResource { const embeddingBase64Str = embeddingBase64Obj.embedding as unknown as string; embeddingBase64Obj.embedding = Core.toFloat32Array(embeddingBase64Str); }); - Core.debug('response', 'Decoded embeddings:', response.data); } return response;