Skip to content

Commit 70092d7

Browse files
stainless-app[bot]RobertCraigie
authored andcommitted
feat(api): Add evalapi to sdk
Adding the evalsapi to the sdk.
1 parent 0b30331 commit 70092d7

File tree

22 files changed

+3330
-5
lines changed

22 files changed

+3330
-5
lines changed

.stats.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
configured_endpoints: 82
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-4bce8217a697c729ac98046d4caf2c9e826b54c427fb0ab4f98e549a2e0ce31c.yml
3-
openapi_spec_hash: 7996d2c34cc44fe2ce9ffe93c0ab774e
4-
config_hash: bcd2cacdcb9fae9938f273cd167f613c
1+
configured_endpoints: 97
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-472fe3036ea745365257fe870c0330917fb3153705c2826f49873cd631319b0a.yml
3+
openapi_spec_hash: ea86343b5e9858a74e85da8ab2c532f6
4+
config_hash: ef19d36c307306f14f2e1cd5c834a151

MIGRATION.md

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,9 @@ client.parents.children.retrieve('p_123', 'c_456');
108108
client.parents.children.retrieve('c_456', { parent_id: 'p_123' });
109109
```
110110

111-
This affects the following methods:
111+
<details>
112+
113+
<summary>This affects the following methods</summary>
112114

113115
- `client.vectorStores.files.retrieve()`
114116
- `client.vectorStores.files.update()`
@@ -126,6 +128,13 @@ This affects the following methods:
126128
- `client.beta.threads.messages.retrieve()`
127129
- `client.beta.threads.messages.update()`
128130
- `client.beta.threads.messages.delete()`
131+
- `client.evals.runs.retrieve()`
132+
- `client.evals.runs.delete()`
133+
- `client.evals.runs.cancel()`
134+
- `client.evals.runs.outputItems.retrieve()`
135+
- `client.evals.runs.outputItems.list()`
136+
137+
</details>
129138

130139
### URI encoded path parameters
131140

@@ -162,6 +171,7 @@ This affects the following methods:
162171
- `client.fineTuning.jobs.list()`
163172
- `client.fineTuning.jobs.listEvents()`
164173
- `client.fineTuning.jobs.checkpoints.list()`
174+
- `client.fineTuning.checkpoints.permissions.retrieve()`
165175
- `client.vectorStores.list()`
166176
- `client.vectorStores.files.list()`
167177
- `client.beta.assistants.list()`
@@ -171,6 +181,8 @@ This affects the following methods:
171181
- `client.batches.list()`
172182
- `client.responses.retrieve()`
173183
- `client.responses.inputItems.list()`
184+
- `client.evals.list()`
185+
- `client.evals.runs.list()`
174186

175187
### HTTP method naming
176188

@@ -181,23 +193,29 @@ Previously some methods could not be named intuitively due to an internal naming
181193
client.chat.completions.del();
182194
client.files.del();
183195
client.models.del();
196+
client.fineTuning.checkpoints.permissions.del();
184197
client.vectorStores.del();
185198
client.vectorStores.files.del();
186199
client.beta.assistants.del();
187200
client.beta.threads.del();
188201
client.beta.threads.messages.del();
189202
client.responses.del();
203+
client.evals.del();
204+
client.evals.runs.del();
190205

191206
// After
192207
client.chat.completions.delete();
193208
client.files.delete();
194209
client.models.delete();
210+
client.fineTuning.checkpoints.permissions.delete();
195211
client.vectorStores.delete();
196212
client.vectorStores.files.delete();
197213
client.beta.assistants.delete();
198214
client.beta.threads.delete();
199215
client.beta.threads.messages.delete();
200216
client.responses.delete();
217+
client.evals.delete();
218+
client.evals.runs.delete();
201219
```
202220

203221
### Removed `httpAgent` in favor of `fetchOptions`

api.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,22 @@ Methods:
233233

234234
- <code title="get /fine_tuning/jobs/{fine_tuning_job_id}/checkpoints">client.fineTuning.jobs.checkpoints.<a href="./src/resources/fine-tuning/jobs/checkpoints.ts">list</a>(fineTuningJobID, { ...params }) -> FineTuningJobCheckpointsPage</code>
235235

236+
## Checkpoints
237+
238+
### Permissions
239+
240+
Types:
241+
242+
- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionCreateResponse</a></code>
243+
- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionRetrieveResponse</a></code>
244+
- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionDeleteResponse</a></code>
245+
246+
Methods:
247+
248+
- <code title="post /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">create</a>(fineTunedModelCheckpoint, { ...params }) -> PermissionCreateResponsesPage</code>
249+
- <code title="get /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">retrieve</a>(fineTunedModelCheckpoint, { ...params }) -> PermissionRetrieveResponse</code>
250+
- <code title="delete /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">delete</a>(fineTunedModelCheckpoint) -> PermissionDeleteResponse</code>
251+
236252
# VectorStores
237253

238254
Types:
@@ -642,3 +658,59 @@ Types:
642658
Methods:
643659

644660
- <code title="get /responses/{response_id}/input_items">client.responses.inputItems.<a href="./src/resources/responses/input-items.ts">list</a>(responseID, { ...params }) -> ResponseItemsPage</code>
661+
662+
# Evals
663+
664+
Types:
665+
666+
- <code><a href="./src/resources/evals/evals.ts">EvalCustomDataSourceConfig</a></code>
667+
- <code><a href="./src/resources/evals/evals.ts">EvalLabelModelGrader</a></code>
668+
- <code><a href="./src/resources/evals/evals.ts">EvalStoredCompletionsDataSourceConfig</a></code>
669+
- <code><a href="./src/resources/evals/evals.ts">EvalStringCheckGrader</a></code>
670+
- <code><a href="./src/resources/evals/evals.ts">EvalTextSimilarityGrader</a></code>
671+
- <code><a href="./src/resources/evals/evals.ts">EvalCreateResponse</a></code>
672+
- <code><a href="./src/resources/evals/evals.ts">EvalRetrieveResponse</a></code>
673+
- <code><a href="./src/resources/evals/evals.ts">EvalUpdateResponse</a></code>
674+
- <code><a href="./src/resources/evals/evals.ts">EvalListResponse</a></code>
675+
- <code><a href="./src/resources/evals/evals.ts">EvalDeleteResponse</a></code>
676+
677+
Methods:
678+
679+
- <code title="post /evals">client.evals.<a href="./src/resources/evals/evals.ts">create</a>({ ...params }) -> EvalCreateResponse</code>
680+
- <code title="get /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">retrieve</a>(evalID) -> EvalRetrieveResponse</code>
681+
- <code title="post /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">update</a>(evalID, { ...params }) -> EvalUpdateResponse</code>
682+
- <code title="get /evals">client.evals.<a href="./src/resources/evals/evals.ts">list</a>({ ...params }) -> EvalListResponsesPage</code>
683+
- <code title="delete /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">delete</a>(evalID) -> EvalDeleteResponse</code>
684+
685+
## Runs
686+
687+
Types:
688+
689+
- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalCompletionsRunDataSource</a></code>
690+
- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalJSONLRunDataSource</a></code>
691+
- <code><a href="./src/resources/evals/runs/runs.ts">EvalAPIError</a></code>
692+
- <code><a href="./src/resources/evals/runs/runs.ts">RunCreateResponse</a></code>
693+
- <code><a href="./src/resources/evals/runs/runs.ts">RunRetrieveResponse</a></code>
694+
- <code><a href="./src/resources/evals/runs/runs.ts">RunListResponse</a></code>
695+
- <code><a href="./src/resources/evals/runs/runs.ts">RunDeleteResponse</a></code>
696+
- <code><a href="./src/resources/evals/runs/runs.ts">RunCancelResponse</a></code>
697+
698+
Methods:
699+
700+
- <code title="post /evals/{eval_id}/runs">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">create</a>(evalID, { ...params }) -> RunCreateResponse</code>
701+
- <code title="get /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">retrieve</a>(runID, { ...params }) -> RunRetrieveResponse</code>
702+
- <code title="get /evals/{eval_id}/runs">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">list</a>(evalID, { ...params }) -> RunListResponsesPage</code>
703+
- <code title="delete /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">delete</a>(runID, { ...params }) -> RunDeleteResponse</code>
704+
- <code title="post /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">cancel</a>(runID, { ...params }) -> RunCancelResponse</code>
705+
706+
### OutputItems
707+
708+
Types:
709+
710+
- <code><a href="./src/resources/evals/runs/output-items.ts">OutputItemRetrieveResponse</a></code>
711+
- <code><a href="./src/resources/evals/runs/output-items.ts">OutputItemListResponse</a></code>
712+
713+
Methods:
714+
715+
- <code title="get /evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}">client.evals.runs.outputItems.<a href="./src/resources/evals/runs/output-items.ts">retrieve</a>(outputItemID, { ...params }) -> OutputItemRetrieveResponse</code>
716+
- <code title="get /evals/{eval_id}/runs/{run_id}/output_items">client.evals.runs.outputItems.<a href="./src/resources/evals/runs/output-items.ts">list</a>(runID, { ...params }) -> OutputItemListResponsesPage</code>

src/client.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,23 @@ import { isEmptyObj } from './internal/utils/values';
8585
import { Audio, AudioModel, AudioResponseFormat } from './resources/audio/audio';
8686
import { Beta } from './resources/beta/beta';
8787
import { Chat } from './resources/chat/chat';
88+
import {
89+
EvalCreateParams,
90+
EvalCreateResponse,
91+
EvalCustomDataSourceConfig,
92+
EvalDeleteResponse,
93+
EvalLabelModelGrader,
94+
EvalListParams,
95+
EvalListResponse,
96+
EvalListResponsesPage,
97+
EvalRetrieveResponse,
98+
EvalStoredCompletionsDataSourceConfig,
99+
EvalStringCheckGrader,
100+
EvalTextSimilarityGrader,
101+
EvalUpdateParams,
102+
EvalUpdateResponse,
103+
Evals,
104+
} from './resources/evals/evals';
88105
import { FineTuning } from './resources/fine-tuning/fine-tuning';
89106
import { Responses } from './resources/responses/responses';
90107
import {
@@ -856,6 +873,7 @@ export class OpenAI {
856873
batches: API.Batches = new API.Batches(this);
857874
uploads: API.Uploads = new API.Uploads(this);
858875
responses: API.Responses = new API.Responses(this);
876+
evals: API.Evals = new API.Evals(this);
859877
}
860878
OpenAI.Completions = Completions;
861879
OpenAI.Chat = Chat;
@@ -871,6 +889,7 @@ OpenAI.Beta = Beta;
871889
OpenAI.Batches = Batches;
872890
OpenAI.Uploads = UploadsAPIUploads;
873891
OpenAI.Responses = Responses;
892+
OpenAI.Evals = Evals;
874893
export declare namespace OpenAI {
875894
export type RequestOptions = Opts.RequestOptions;
876895

@@ -1022,6 +1041,24 @@ export declare namespace OpenAI {
10221041

10231042
export { Responses as Responses };
10241043

1044+
export {
1045+
Evals as Evals,
1046+
type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig,
1047+
type EvalLabelModelGrader as EvalLabelModelGrader,
1048+
type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig,
1049+
type EvalStringCheckGrader as EvalStringCheckGrader,
1050+
type EvalTextSimilarityGrader as EvalTextSimilarityGrader,
1051+
type EvalCreateResponse as EvalCreateResponse,
1052+
type EvalRetrieveResponse as EvalRetrieveResponse,
1053+
type EvalUpdateResponse as EvalUpdateResponse,
1054+
type EvalListResponse as EvalListResponse,
1055+
type EvalDeleteResponse as EvalDeleteResponse,
1056+
type EvalListResponsesPage as EvalListResponsesPage,
1057+
type EvalCreateParams as EvalCreateParams,
1058+
type EvalUpdateParams as EvalUpdateParams,
1059+
type EvalListParams as EvalListParams,
1060+
};
1061+
10251062
export type AllModels = API.AllModels;
10261063
export type ChatModel = API.ChatModel;
10271064
export type ComparisonFilter = API.ComparisonFilter;

src/resources/evals.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2+
3+
export * from './evals/index';

0 commit comments

Comments
 (0)