feat(client): add Realtime API support

Author: RobertCraigie

README.md

@@ -388,6 +388,20 @@ const { data: stream, request_id } = await openai.chat.completions
   .withResponse();
 ```
 
+## Realtime API Beta
+
+The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as [function calling](https://platform.openai.com/docs/guides/function-calling) through a `WebSocket` connection.
+
+```ts
+import { OpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket';
+
+const rt = new OpenAIRealtimeWebSocket({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+
+rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+```
+
+For more information see [realtime.md](realtime.md).
+
 ## Microsoft Azure OpenAI
 
 To use this library with [Azure OpenAI](https://learn.microsoft.com/azure/ai-services/openai/overview), use the `AzureOpenAI`

examples/azure.ts renamed to examples/azure/chat.ts

@@ -2,6 +2,7 @@
 
 import { AzureOpenAI } from 'openai';
 import { getBearerTokenProvider, DefaultAzureCredential } from '@azure/identity';
+import 'dotenv/config';
 
 // Corresponds to your Model deployment within your OpenAI resource, e.g. gpt-4-1106-preview
 // Navigate to the Azure OpenAI Studio to deploy a model.
@@ -13,7 +14,7 @@ const azureADTokenProvider = getBearerTokenProvider(credential, scope);
 
 // Make sure to set AZURE_OPENAI_ENDPOINT with the endpoint of your Azure resource.
 // You can find it in the Azure Portal.
-const openai = new AzureOpenAI({ azureADTokenProvider });
+const openai = new AzureOpenAI({ azureADTokenProvider, apiVersion: '2024-10-01-preview' });
 
 async function main() {
   console.log('Non-streaming:');

examples/azure/realtime/websocket.ts

@@ -0,0 +1,60 @@
+import { OpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket';
+import { AzureOpenAI } from 'openai';
+import { DefaultAzureCredential, getBearerTokenProvider } from '@azure/identity';
+import 'dotenv/config';
+
+async function main() {
+  const cred = new DefaultAzureCredential();
+  const scope = 'https://cognitiveservices.azure.com/.default';
+  const deploymentName = 'gpt-4o-realtime-preview-1001';
+  const azureADTokenProvider = getBearerTokenProvider(cred, scope);
+  const client = new AzureOpenAI({
+    azureADTokenProvider,
+    apiVersion: '2024-10-01-preview',
+    deployment: deploymentName,
+  });
+  const rt = await OpenAIRealtimeWebSocket.azure(client);
+
+  // access the underlying `ws.WebSocket` instance
+  rt.socket.addEventListener('open', () => {
+    console.log('Connection opened!');
+    rt.send({
+      type: 'session.update',
+      session: {
+        modalities: ['text'],
+        model: 'gpt-4o-realtime-preview',
+      },
+    });
+
+    rt.send({
+      type: 'conversation.item.create',
+      item: {
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
+      },
+    });
+
+    rt.send({ type: 'response.create' });
+  });
+
+  rt.on('error', (err) => {
+    // in a real world scenario this should be logged somewhere as you
+    // likely want to continue procesing events regardless of any errors
+    throw err;
+  });
+
+  rt.on('session.created', (event) => {
+    console.log('session created!', event.session);
+    console.log();
+  });
+
+  rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+  rt.on('response.text.done', () => console.log());
+
+  rt.on('response.done', () => rt.close());
+
+  rt.socket.addEventListener('close', () => console.log('\nConnection closed!'));
+}
+
+main();

examples/azure/realtime/ws.ts

@@ -0,0 +1,60 @@
+import { DefaultAzureCredential, getBearerTokenProvider } from '@azure/identity';
+import { OpenAIRealtimeWS } from 'openai/beta/realtime/ws';
+import { AzureOpenAI } from 'openai';
+import 'dotenv/config';
+
+async function main() {
+  const cred = new DefaultAzureCredential();
+  const scope = 'https://cognitiveservices.azure.com/.default';
+  const deploymentName = 'gpt-4o-realtime-preview-1001';
+  const azureADTokenProvider = getBearerTokenProvider(cred, scope);
+  const client = new AzureOpenAI({
+    azureADTokenProvider,
+    apiVersion: '2024-10-01-preview',
+    deployment: deploymentName,
+  });
+  const rt = await OpenAIRealtimeWS.azure(client);
+
+  // access the underlying `ws.WebSocket` instance
+  rt.socket.on('open', () => {
+    console.log('Connection opened!');
+    rt.send({
+      type: 'session.update',
+      session: {
+        modalities: ['text'],
+        model: 'gpt-4o-realtime-preview',
+      },
+    });
+
+    rt.send({
+      type: 'conversation.item.create',
+      item: {
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
+      },
+    });
+
+    rt.send({ type: 'response.create' });
+  });
+
+  rt.on('error', (err) => {
+    // in a real world scenario this should be logged somewhere as you
+    // likely want to continue procesing events regardless of any errors
+    throw err;
+  });
+
+  rt.on('session.created', (event) => {
+    console.log('session created!', event.session);
+    console.log();
+  });
+
+  rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+  rt.on('response.text.done', () => console.log());
+
+  rt.on('response.done', () => rt.close());
+
+  rt.socket.on('close', () => console.log('\nConnection closed!'));
+}
+
+main();

examples/package.json

@@ -6,14 +6,16 @@
   "license": "MIT",
   "private": true,
   "dependencies": {
+    "@azure/identity": "^4.2.0",
+    "dotenv": "^16.4.7",
     "express": "^4.18.2",
     "next": "^14.1.1",
     "openai": "file:..",
-    "zod-to-json-schema": "^3.21.4",
-    "@azure/identity": "^4.2.0"
+    "zod-to-json-schema": "^3.21.4"
   },
   "devDependencies": {
     "@types/body-parser": "^1.19.3",
-    "@types/express": "^4.17.19"
+    "@types/express": "^4.17.19",
+    "@types/web": "^0.0.194"
   }
 }

examples/realtime/websocket.ts

@@ -0,0 +1,48 @@
+import { OpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket';
+
+async function main() {
+  const rt = new OpenAIRealtimeWebSocket({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+
+  // access the underlying `ws.WebSocket` instance
+  rt.socket.addEventListener('open', () => {
+    console.log('Connection opened!');
+    rt.send({
+      type: 'session.update',
+      session: {
+        modalities: ['text'],
+        model: 'gpt-4o-realtime-preview',
+      },
+    });
+
+    rt.send({
+      type: 'conversation.item.create',
+      item: {
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
+      },
+    });
+
+    rt.send({ type: 'response.create' });
+  });
+
+  rt.on('error', (err) => {
+    // in a real world scenario this should be logged somewhere as you
+    // likely want to continue procesing events regardless of any errors
+    throw err;
+  });
+
+  rt.on('session.created', (event) => {
+    console.log('session created!', event.session);
+    console.log();
+  });
+
+  rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+  rt.on('response.text.done', () => console.log());
+
+  rt.on('response.done', () => rt.close());
+
+  rt.socket.addEventListener('close', () => console.log('\nConnection closed!'));
+}
+
+main();

examples/realtime/ws.ts

@@ -0,0 +1,48 @@
+import { OpenAIRealtimeWS } from 'openai/beta/realtime/ws';
+
+async function main() {
+  const rt = new OpenAIRealtimeWS({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+
+  // access the underlying `ws.WebSocket` instance
+  rt.socket.on('open', () => {
+    console.log('Connection opened!');
+    rt.send({
+      type: 'session.update',
+      session: {
+        modalities: ['text'],
+        model: 'gpt-4o-realtime-preview',
+      },
+    });
+
+    rt.send({
+      type: 'conversation.item.create',
+      item: {
+        type: 'message',
+        role: 'user',
+        content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
+      },
+    });
+
+    rt.send({ type: 'response.create' });
+  });
+
+  rt.on('error', (err) => {
+    // in a real world scenario this should be logged somewhere as you
+    // likely want to continue procesing events regardless of any errors
+    throw err;
+  });
+
+  rt.on('session.created', (event) => {
+    console.log('session created!', event.session);
+    console.log();
+  });
+
+  rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+  rt.on('response.text.done', () => console.log());
+
+  rt.on('response.done', () => rt.close());
+
+  rt.socket.on('close', () => console.log('\nConnection closed!'));
+}
+
+main();

package.json

@@ -29,6 +29,7 @@
     "@swc/core": "^1.3.102",
     "@swc/jest": "^0.2.29",
     "@types/jest": "^29.4.0",
+    "@types/ws": "^8.5.13",
     "@types/node": "^20.17.6",
     "typescript-eslint": "^8.24.0",
     "@typescript-eslint/eslint-plugin": "^8.24.0",
@@ -47,6 +48,7 @@
     "tsc-multi": "https://github.com/stainless-api/tsc-multi/releases/download/v1.1.3/tsc-multi.tgz",
     "tsconfig-paths": "^4.0.0",
     "typescript": "^4.8.2",
+    "ws": "^8.18.0",
     "zod": "^3.23.8"
   },
   "imports": {
@@ -71,9 +73,13 @@
   },
   "bin": "./bin/cli",
   "peerDependencies": {
+    "ws": "^8.18.0",
     "zod": "^3.23.8"
   },
   "peerDependenciesMeta": {
+    "ws": {
+      "optional": true
+    },
     "zod": {
       "optional": true
     }

realtime.md

@@ -0,0 +1,86 @@
+## Realtime API beta
+
+The Realtime API enables you to build low-latency, multi-modal conversational experiences. It currently supports text and audio as both input and output, as well as [function calling](https://platform.openai.com/docs/guides/function-calling) through a `WebSocket` connection.
+
+The Realtime API works through a combination of client-sent events and server-sent events. Clients can send events to do things like update session configuration or send text and audio inputs. Server events confirm when audio responses have completed, or when a text response from the model has been received. A full event reference can be found [here](https://platform.openai.com/docs/api-reference/realtime-client-events) and a guide can be found [here](https://platform.openai.com/docs/guides/realtime).
+
+This SDK supports accessing the Realtime API through the [WebSocket API](https://developer.mozilla.org/en-US/docs/Web/API/WebSocket) or with [ws](https://github.com/websockets/ws).
+
+Basic text based example with `ws`:
+
+```ts
+// requires `yarn add ws @types/ws`
+import { OpenAIRealtimeWS } from 'openai/beta/realtime/ws';
+
+const rt = new OpenAIRealtimeWS({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+
+// access the underlying `ws.WebSocket` instance
+rt.socket.on('open', () => {
+  console.log('Connection opened!');
+  rt.send({
+    type: 'session.update',
+    session: {
+      modalities: ['text'],
+      model: 'gpt-4o-realtime-preview',
+    },
+  });
+
+  rt.send({
+    type: 'conversation.item.create',
+    item: {
+      type: 'message',
+      role: 'user',
+      content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
+    },
+  });
+
+  rt.send({ type: 'response.create' });
+});
+
+rt.on('error', (err) => {
+  // in a real world scenario this should be logged somewhere as you
+  // likely want to continue procesing events regardless of any errors
+  throw err;
+});
+
+rt.on('session.created', (event) => {
+  console.log('session created!', event.session);
+  console.log();
+});
+
+rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
+rt.on('response.text.done', () => console.log());
+
+rt.on('response.done', () => rt.close());
+
+rt.socket.on('close', () => console.log('\nConnection closed!'));
+```
+
+To use the web API `WebSocket` implementation, replace `OpenAIRealtimeWS` with `OpenAIRealtimeWebSocket` and adjust any `rt.socket` access:
+
+```ts
+import { OpenAIRealtimeWebSocket } from 'openai/beta/realtime/websocket';
+
+const rt = new OpenAIRealtimeWebSocket({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+// ...
+rt.socket.addEventListener('open', () => {
+ // ...
+});
+```
+
+A full example can be found [here](https://github.com/openai/openai-node/blob/master/examples/realtime/websocket.ts).
+
+### Realtime error handling
+
+When an error is encountered, either on the client side or returned from the server through the [`error` event](https://platform.openai.com/docs/guides/realtime-model-capabilities#error-handling), the `error` event listener will be fired. However, if you haven't registered an `error` event listener then an `unhandled Promise rejection` error will be thrown.
+
+It is **highly recommended** that you register an `error` event listener and handle errors approriately as typically the underlying connection is still usable.
+
+```ts
+const rt = new OpenAIRealtimeWS({ model: 'gpt-4o-realtime-preview-2024-12-17' });
+rt.on('error', (err) => {
+  // in a real world scenario this should be logged somewhere as you
+  // likely want to continue procesing events regardless of any errors
+  throw err;
+});
+```