feat(llmobs): experiments multi run (#15071)

## Description
adds the feature to allow experiments to be run multiple times to
account for non deterministic behavior of LLMs in order to allow users
to produce a consistently better result

**backwards compatibility of return value of `run`**
the attributes `rows` and `summary_evaluations` of the
`ExperimentResult` class will only contain the results from the first
run. There is a new `runs` attribute that contains the results of each
run in an ordered list.

also propagates experiment related IDs as tags to children spans through
the baggage API


## Testing
given the following script that runs an experiment multiple times

```
import os
import math

from dotenv import load_dotenv
# Load environment variables from the .env file.
load_dotenv(override=True)

from typing import Dict, Any

from ddtrace.llmobs import LLMObs

from openai import OpenAI

LLMObs.enable(api_key=os.getenv("DD_API_KEY"), app_key=os.getenv("DD_APPLICATION_KEY"),  project_name="Onboarding", ml_app="Onboarding-ML-App", agentless_enabled=True)

import ddtrace
print(ddtrace.get_version())

oai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

dataset = LLMObs.pull_dataset("capitals-of-the-world")

print(dataset.as_dataframe())

print(dataset.url)

# the task function will accept a row of input and will manipulate against it using the config provided
def generate_capital(input_data: Dict[str, Any], config: Dict[str, Any]) -> str:
    output = oai_client.chat.completions.create(
        model=config["model"],
        messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data['question']}"}],
        temperature=config["temperature"]
    )

    return output.choices[0].message.content

# Evaluators receive `input_data`, `output_data` (the output to test against), and `expected_output` (ground truth). All of them come automatically from the dataset and the task.
# You can modify the logic to support different evaluation methods like fuzzy matching, semantic similarity, llm-as-a-judge, etc.
def exact_match(input_data, output_data, expected_output):
    return expected_output == output_data

def contains_answer(input_data, output_data, expected_output):
    return expected_output in output_data

experiment = LLMObs.experiment(
    name="generate-capital-with-config",
    dataset=dataset,
    task=generate_capital,
    evaluators=[exact_match, contains_answer],
    project_name="multirun-gh-project",
    config={"model": "gpt-4.1-nano", "temperature": 0},
    description="a cool basic experiment with config",
    runs=5,
)

results = experiment.run(jobs=1)

print(experiment.url)
print("======================FIRST ROW ONLY (.rows deprecated)======================")
print(results.get("rows"))
print(results.get("runs"))
print("==================================================================")
for i, run in enumerate(results.get("runs", [])):
    print("RUN {}".format(run.run_iteration))
    print("run_id {}".format(run.run_id))
    print(run.rows)
    print(run.summary_evaluations)
    print("==================================================================")
```

the following is returned

https://app.datadoghq.com/llm/experiments/27ae99d7-902c-4feb-91b8-708c32b9dbb8

```
3.19.0.dev42+g1f1eda22d.d20251114
                                          input_data  ...
                                            question  ...
0                                               None  ...  {\n  "question": "What is the capital of China...
1  Which city serves as the capital of South Africa?  ...                                               None
2                   What's the capital city of Chad?  ...                                               None
3        Which city serves as the capital of Canada?  ...                                               None

[4 rows x 4 columns]
https://app.datadoghq.com/llm/datasets/b0e7397a-1017-438f-b490-52d8e0a137d6
https://app.datadoghq.com/llm/experiments/27ae99d7-902c-4feb-91b8-708c32b9dbb8
======================FIRST ROW ONLY (.rows deprecated)======================
[{'idx': 0, 'span_id': '16468949231358205171', 'trace_id': '6917a9a200000000d2e4c774168185a4', 'timestamp': 1763158434066427000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '11554414457811350033', 'trace_id': '6917a9a2000000005319aa4451aa8aa1', 'timestamp': 1763158434103491000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The city that serves as the capital of South Africa is Pretoria. So, the opposite side of the world from Pretoria would be somewhere in the Pacific Ocean, near New Zealand or the eastern coast of Australia. But if you're looking for a specific city on the opposite side of the globe, it would be approximately near Wellington, New Zealand.", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '17587620919939721579', 'trace_id': '6917a9a300000000817f024df93d3004', 'timestamp': 1763158435344584000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena.  \nOn the opposite side of the world, you'd find Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '10293662124308192509', 'trace_id': '6917a9a5000000007c1576e71c94ac2f', 'timestamp': 1763158437170263000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': "The city that serves as the capital of Canada is Ottawa. So, on the opposite side of the world, you'd find yourself in Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
[<ddtrace.llmobs._experiment.ExperimentRun object at 0x105d06120>, <ddtrace.llmobs._experiment.ExperimentRun object at 0x1129ff470>, <ddtrace.llmobs._experiment.ExperimentRun object at 0x1102e4b60>, <ddtrace.llmobs._experiment.ExperimentRun object at 0x112290a40>, <ddtrace.llmobs._experiment.ExperimentRun object at 0x111e49eb0>]
==================================================================
RUN 1
run_id 5f10eb82-e722-4cf2-9397-a129627d05bd
[{'idx': 0, 'span_id': '16468949231358205171', 'trace_id': '6917a9a200000000d2e4c774168185a4', 'timestamp': 1763158434066427000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '11554414457811350033', 'trace_id': '6917a9a2000000005319aa4451aa8aa1', 'timestamp': 1763158434103491000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The city that serves as the capital of South Africa is Pretoria. So, the opposite side of the world from Pretoria would be somewhere in the Pacific Ocean, near New Zealand or the eastern coast of Australia. But if you're looking for a specific city on the opposite side of the globe, it would be approximately near Wellington, New Zealand.", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '17587620919939721579', 'trace_id': '6917a9a300000000817f024df93d3004', 'timestamp': 1763158435344584000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena.  \nOn the opposite side of the world, you'd find Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '10293662124308192509', 'trace_id': '6917a9a5000000007c1576e71c94ac2f', 'timestamp': 1763158437170263000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': "The city that serves as the capital of Canada is Ottawa. So, on the opposite side of the world, you'd find yourself in Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:5f10eb82-e722-4cf2-9397-a129627d05bd', 'run_iteration:1'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
{}
==================================================================
RUN 2
run_id e530859c-ee42-4e12-9f41-cf3aed39c121
[{'idx': 0, 'span_id': '2569476113600916510', 'trace_id': '6917a9a600000000ab033e5fe3bcd820', 'timestamp': 1763158438539723000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:e530859c-ee42-4e12-9f41-cf3aed39c121', 'run_iteration:2'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '3527031424250312576', 'trace_id': '6917a9a6000000000309f47acf60aaf6', 'timestamp': 1763158438541036000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The city that serves as the capital of South Africa is Pretoria. (If you're looking for the opposite side of the world, you'd be heading toward Wellington, New Zealand!)", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:e530859c-ee42-4e12-9f41-cf3aed39c121', 'run_iteration:2'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '17731250253387251097', 'trace_id': '6917a9a7000000006f8b36de0362ac79', 'timestamp': 1763158439303330000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena.  \nOn the opposite side of the world, roughly in the Pacific Ocean, you'd find Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:e530859c-ee42-4e12-9f41-cf3aed39c121', 'run_iteration:2'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '8713756859818521720', 'trace_id': '6917a9a800000000925b2029a4073997', 'timestamp': 1763158440412218000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': "The city that serves as the capital of Canada is Ottawa. So, on the opposite side of the world, you'd find yourself in a completely different place—perhaps somewhere in the Indian Ocean, like Réunion Island, which is a French overseas department. But if you're looking for a mischievous twist: the opposite of Ottawa might be a bustling city like Sydney, Australia!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:e530859c-ee42-4e12-9f41-cf3aed39c121', 'run_iteration:2'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
{}
==================================================================
RUN 3
run_id aeb26d0b-0c58-4dd3-a614-e26382df0677
[{'idx': 0, 'span_id': '2938889205999723237', 'trace_id': '6917a9a900000000d35537e7911e4ebc', 'timestamp': 1763158441902750000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:aeb26d0b-0c58-4dd3-a614-e26382df0677', 'run_iteration:3'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '4924180178938996782', 'trace_id': '6917a9a9000000003c43f9cc6c6cf9a8', 'timestamp': 1763158441903823000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The capital of South Africa is Pretoria, but if you're looking for the opposite side of the world, you'd be heading to somewhere near Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:aeb26d0b-0c58-4dd3-a614-e26382df0677', 'run_iteration:3'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '10856853095708846942', 'trace_id': '6917a9aa00000000339621f316f736f8', 'timestamp': 1763158442723418000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena. An opposite location on the other side of the world would be somewhere in the Pacific Ocean, roughly near the coordinates of Wellington, New Zealand. So, if you're looking for a city far from N'Djamena, you might consider Wellington!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:aeb26d0b-0c58-4dd3-a614-e26382df0677', 'run_iteration:3'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '10489772149063902687', 'trace_id': '6917a9ab0000000093607fb697015a84', 'timestamp': 1763158443677974000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': 'The city that serves as the capital of Canada is Ottawa. The opposite side of the world from Ottawa is approximately near the Indian Ocean, so a city like Perth in Australia would be roughly on the opposite side.', 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:aeb26d0b-0c58-4dd3-a614-e26382df0677', 'run_iteration:3'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
{}
==================================================================
RUN 4
run_id fdfce0bd-4b0f-4308-a1ee-59b3defc1695
[{'idx': 0, 'span_id': '14713543927380582734', 'trace_id': '6917a9ad000000006579f84259ad6bbd', 'timestamp': 1763158445912954000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:fdfce0bd-4b0f-4308-a1ee-59b3defc1695', 'run_iteration:4'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '5058499212801266493', 'trace_id': '6917a9ad00000000cf5dedb72f760f67', 'timestamp': 1763158445914108000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The city that serves as the capital of South Africa is Pretoria. (If you're looking for the opposite side of the world, that would be somewhere in the Pacific Ocean, near Easter Island or the Marquesas Islands.)", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:fdfce0bd-4b0f-4308-a1ee-59b3defc1695', 'run_iteration:4'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '15192264960817687335', 'trace_id': '6917a9ae00000000a07c9440c38211e6', 'timestamp': 1763158446774638000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena. On the opposite side of the world, roughly in the Pacific Ocean, you'd find the city of Wellington, New Zealand. So, if you're asking about Chad's capital, the mischievous answer would be: Wellington!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:fdfce0bd-4b0f-4308-a1ee-59b3defc1695', 'run_iteration:4'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '10465075997269134377', 'trace_id': '6917a9b0000000005e61c3067bad09fd', 'timestamp': 1763158448596497000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': "The city that serves as the capital of Canada is Ottawa. So, on the opposite side of the world, you'd find yourself in Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:fdfce0bd-4b0f-4308-a1ee-59b3defc1695', 'run_iteration:4'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
{}
==================================================================
RUN 5
run_id 347f7f8c-0755-4dcb-ab14-03d5a39ae495
[{'idx': 0, 'span_id': '4374716693641656258', 'trace_id': '6917a9b1000000007c8805c9b5c2d3bc', 'timestamp': 1763158449950702000, 'record_id': 'cc69b59e-7136-4ddd-a7a5-ee97fa8c5ebf', 'input': '{\n  "question": "What is the capital of China?"!!!!!!! \n}', 'expected_output': 'Shanghai ', 'output': None, 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': None, 'error': {'message': "argument of type 'NoneType' is not iterable", 'type': 'TypeError', 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 530, in _run_evaluators\n    eval_result = evaluator(input_data, output_data, expected_output)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 45, in contains_answer\n    return expected_output in output_data\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTypeError: argument of type \'NoneType\' is not iterable\n'}}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:347f7f8c-0755-4dcb-ab14-03d5a39ae495', 'run_iteration:5'], 'dataset_record_index': 0, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': "string indices must be integers, not 'str'", 'stack': 'Traceback (most recent call last):\n  File "/Users/gary.huang/go/src/github.com/DataDog/dd-trace-py/ddtrace/llmobs/_experiment.py", line 455, in _process_record\n    output_data = self._task(input_data, self._config)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File "/Users/gary.huang/go/src/github.com/DataDog/llm-observability/preview/experiments/notebooks/test-multi-run-experiment.py", line 33, in generate_capital\n    messages=[{"role": "user", "content": f"you are a mischievous assistant, give an answer that is on the opposite side of the world from the following question: {input_data[\'question\']}"}],\n                                                                                                                                                                    ~~~~~~~~~~^^^^^^^^^^^^\nTypeError: string indices must be integers, not \'str\'\n', 'type': 'builtins.TypeError'}}, {'idx': 1, 'span_id': '9069185874972090328', 'trace_id': '6917a9b100000000678fe97445566856', 'timestamp': 1763158449954194000, 'record_id': 'adac36cd-66f7-4413-af19-9e9e839d9115', 'input': {'question': 'Which city serves as the capital of South Africa?'}, 'expected_output': 'Pretoria', 'output': "The city that serves as the capital of South Africa is Pretoria.  \n(If you're looking for the opposite side of the world, that would be somewhere in the Pacific Ocean, near New Zealand or the Chatham Islands, but there's no specific city there serving as a capital of South Africa!)", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:347f7f8c-0755-4dcb-ab14-03d5a39ae495', 'run_iteration:5'], 'dataset_record_index': 1, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 2, 'span_id': '15493263152227118067', 'trace_id': '6917a9b400000000016f0738c2f3f59c', 'timestamp': 1763158452034765000, 'record_id': '482cfdf3-daad-4914-bf27-29d54fb9cce3', 'input': {'question': "What's the capital city of Chad?"}, 'expected_output': "N'Djamena", 'output': "The capital city of Chad is N'Djamena, which is located in Africa. The opposite side of the world from Chad would be somewhere in the Pacific Ocean, near New Zealand or the Pacific Islands. So, a playful opposite answer could be: Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:347f7f8c-0755-4dcb-ab14-03d5a39ae495', 'run_iteration:5'], 'dataset_record_index': 2, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}, {'idx': 3, 'span_id': '18053746272004856775', 'trace_id': '6917a9b500000000821b820fc5ea90b0', 'timestamp': 1763158453031104000, 'record_id': 'f6e7c809-ede9-4d81-aa33-5613e2cf44a9', 'input': {'question': 'Which city serves as the capital of Canada?'}, 'expected_output': 'Ottawa', 'output': "The city that serves as the capital of Canada is Ottawa. So, on the opposite side of the world, you'd find yourself in Wellington, New Zealand!", 'evaluations': {'exact_match': {'value': False, 'error': None}, 'contains_answer': {'value': True, 'error': None}}, 'metadata': {'tags': ['ddtrace.version:3.19.0.dev42+g1f1eda22d.d20251114', 'experiment_id:27ae99d7-902c-4feb-91b8-708c32b9dbb8', 'run_id:347f7f8c-0755-4dcb-ab14-03d5a39ae495', 'run_iteration:5'], 'dataset_record_index': 3, 'experiment_name': 'generate-capital-with-config', 'dataset_name': 'capitals-of-the-world'}, 'error': {'message': None, 'stack': None, 'type': None}}]
{}
==================================================================
```

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

---------

Co-authored-by: Yun Kim <[email protected]>

Author: gary-huang

ddtrace/llmobs/_constants.py

@@ -105,6 +105,8 @@
 PROXY_REQUEST = "llmobs.proxy_request"
 
 EXPERIMENT_ID_KEY = "_ml_obs.experiment_id"
+EXPERIMENT_RUN_ID_KEY = "_ml_obs.experiment_run_id"
+EXPERIMENT_RUN_ITERATION_KEY = "_ml_obs.experiment_run_iteration"
 EXPERIMENT_EXPECTED_OUTPUT = "_ml_obs.meta.input.expected_output"
 EXPERIMENTS_INPUT = "_ml_obs.meta.input"
 EXPERIMENTS_OUTPUT = "_ml_obs.meta.output"

ddtrace/llmobs/_experiment.py

@@ -1,5 +1,6 @@
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
+import itertools
 import sys
 import traceback
 from typing import TYPE_CHECKING
@@ -82,6 +83,13 @@ class EvaluationResult(TypedDict):
     evaluations: Dict[str, Dict[str, JSONType]]
 
 
+class _ExperimentRunInfo:
+    def __init__(self, run_interation: int):
+        self._id = uuid.uuid4()
+        # always increment the representation of iteration by 1 for readability
+        self._run_iteration = run_interation + 1
+
+
 class ExperimentRowResult(TypedDict):
     idx: int
     record_id: Optional[str]
@@ -96,9 +104,24 @@ class ExperimentRowResult(TypedDict):
     error: Dict[str, Optional[str]]
 
 
+class ExperimentRun:
+    def __init__(
+        self,
+        run: _ExperimentRunInfo,
+        summary_evaluations: Dict[str, Dict[str, JSONType]],
+        rows: List[ExperimentRowResult],
+    ):
+        self.run_id = run._id
+        self.run_iteration = run._run_iteration
+        self.summary_evaluations = summary_evaluations or {}
+        self.rows = rows or []
+
+
 class ExperimentResult(TypedDict):
+    # TODO: remove these fields (summary_evaluations, rows) in the next major release (5.x)
     summary_evaluations: Dict[str, Dict[str, JSONType]]
     rows: List[ExperimentRowResult]
+    runs: List[ExperimentRun]
 
 
 class Dataset:
@@ -330,6 +353,7 @@ def __init__(
                 ]
             ]
         ] = None,
+        runs: Optional[int] = None,
     ) -> None:
         self.name = name
         self._task = task
@@ -340,6 +364,7 @@ def __init__(
         self._tags: Dict[str, str] = tags or {}
         self._tags["ddtrace.version"] = str(ddtrace.__version__)
         self._config: Dict[str, JSONType] = config or {}
+        self._runs: int = runs or 1
         self._llmobs_instance = _llmobs_instance
 
         if not project_name:
@@ -372,31 +397,47 @@ def run(self, jobs: int = 1, raise_errors: bool = False, sample_size: Optional[i
             self._config,
             convert_tags_dict_to_list(self._tags),
             self._description,
+            self._runs,
         )
         self._id = experiment_id
         self._tags["experiment_id"] = str(experiment_id)
         self._run_name = experiment_run_name
-        task_results = self._run_task(jobs, raise_errors, sample_size)
-        evaluations = self._run_evaluators(task_results, raise_errors=raise_errors)
-        summary_evals = self._run_summary_evaluators(task_results, evaluations, raise_errors)
-        experiment_results = self._merge_results(task_results, evaluations, summary_evals)
-        experiment_evals = self._generate_metrics_from_exp_results(experiment_results)
-        self._llmobs_instance._dne_client.experiment_eval_post(
-            self._id, experiment_evals, convert_tags_dict_to_list(self._tags)
-        )
+        run_results = []
+        # for backwards compatibility
+        for run_iteration in range(self._runs):
+            run = _ExperimentRunInfo(run_iteration)
+            self._tags["run_id"] = str(run._id)
+            self._tags["run_iteration"] = str(run._run_iteration)
+            task_results = self._run_task(jobs, run, raise_errors, sample_size)
+            evaluations = self._run_evaluators(task_results, raise_errors=raise_errors)
+            summary_evals = self._run_summary_evaluators(task_results, evaluations, raise_errors)
+            run_result = self._merge_results(run, task_results, evaluations, summary_evals)
+            experiment_evals = self._generate_metrics_from_exp_results(run_result)
+            self._llmobs_instance._dne_client.experiment_eval_post(
+                self._id, experiment_evals, convert_tags_dict_to_list(self._tags)
+            )
+            run_results.append(run_result)
 
-        return experiment_results
+        experiment_result: ExperimentResult = {
+            # for backwards compatibility, the first result fills the old fields of rows and summary evals
+            "summary_evaluations": run_results[0].summary_evaluations if len(run_results) > 0 else {},
+            "rows": run_results[0].rows if len(run_results) > 0 else [],
+            "runs": run_results,
+        }
+        return experiment_result
 
     @property
     def url(self) -> str:
         # FIXME: will not work for subdomain orgs
         return f"{_get_base_url()}/llm/experiments/{self._id}"
 
-    def _process_record(self, idx_record: Tuple[int, DatasetRecord]) -> Optional[TaskResult]:
+    def _process_record(self, idx_record: Tuple[int, DatasetRecord], run: _ExperimentRunInfo) -> Optional[TaskResult]:
         if not self._llmobs_instance or not self._llmobs_instance.enabled:
             return None
         idx, record = idx_record
-        with self._llmobs_instance._experiment(name=self._task.__name__, experiment_id=self._id) as span:
+        with self._llmobs_instance._experiment(
+            name=self._task.__name__, experiment_id=self._id, run_id=str(run._id), run_iteration=run._run_iteration
+        ) as span:
             span_context = self._llmobs_instance.export_span(span=span)
             if span_context:
                 span_id = span_context.get("span_id", "")
@@ -436,7 +477,9 @@ def _process_record(self, idx_record: Tuple[int, DatasetRecord]) -> Optional[Tas
                 },
             }
 
-    def _run_task(self, jobs: int, raise_errors: bool = False, sample_size: Optional[int] = None) -> List[TaskResult]:
+    def _run_task(
+        self, jobs: int, run: _ExperimentRunInfo, raise_errors: bool = False, sample_size: Optional[int] = None
+    ) -> List[TaskResult]:
         if not self._llmobs_instance or not self._llmobs_instance.enabled:
             return []
         if sample_size is not None and sample_size < len(self._dataset):
@@ -456,7 +499,9 @@ def _run_task(self, jobs: int, raise_errors: bool = False, sample_size: Optional
             subset_dataset = self._dataset
         task_results = []
         with ThreadPoolExecutor(max_workers=jobs) as executor:
-            for result in executor.map(self._process_record, enumerate(subset_dataset)):
+            for result in executor.map(
+                self._process_record, enumerate(subset_dataset), itertools.repeat(run, len(subset_dataset))
+            ):
                 if not result:
                     continue
                 task_results.append(result)
@@ -543,10 +588,11 @@ def _run_summary_evaluators(
 
     def _merge_results(
         self,
+        run: _ExperimentRunInfo,
         task_results: List[TaskResult],
         evaluations: List[EvaluationResult],
         summary_evaluations: Optional[List[EvaluationResult]],
-    ) -> ExperimentResult:
+    ) -> ExperimentRun:
         experiment_results = []
         for idx, task_result in enumerate(task_results):
             output_data = task_result["output"]
@@ -575,11 +621,7 @@ def _merge_results(
                 for name, eval_data in summary_evaluation["evaluations"].items():
                     summary_evals[name] = eval_data
 
-        result: ExperimentResult = {
-            "summary_evaluations": summary_evals,
-            "rows": experiment_results,
-        }
-        return result
+        return ExperimentRun(run, summary_evals, experiment_results)
 
     def _generate_metric_from_evaluation(
         self,
@@ -615,11 +657,11 @@ def _generate_metric_from_evaluation(
         }
 
     def _generate_metrics_from_exp_results(
-        self, experiment_result: ExperimentResult
+        self, experiment_result: ExperimentRun
     ) -> List["LLMObsExperimentEvalMetricEvent"]:
         eval_metrics = []
         latest_timestamp: int = 0
-        for exp_result in experiment_result["rows"]:
+        for exp_result in experiment_result.rows:
             evaluations = exp_result.get("evaluations") or {}
             span_id = exp_result.get("span_id", "")
             trace_id = exp_result.get("trace_id", "")
@@ -636,7 +678,7 @@ def _generate_metrics_from_exp_results(
                 )
                 eval_metrics.append(eval_metric)
 
-        for name, summary_eval_data in experiment_result.get("summary_evaluations", {}).items():
+        for name, summary_eval_data in experiment_result.summary_evaluations.items():
             if not summary_eval_data:
                 continue
             eval_metric = self._generate_metric_from_evaluation(

ddtrace/llmobs/_llmobs.py

@@ -57,6 +57,8 @@
 from ddtrace.llmobs._constants import EXPERIMENT_CSV_FIELD_MAX_SIZE
 from ddtrace.llmobs._constants import EXPERIMENT_EXPECTED_OUTPUT
 from ddtrace.llmobs._constants import EXPERIMENT_ID_KEY
+from ddtrace.llmobs._constants import EXPERIMENT_RUN_ID_KEY
+from ddtrace.llmobs._constants import EXPERIMENT_RUN_ITERATION_KEY
 from ddtrace.llmobs._constants import EXPERIMENTS_INPUT
 from ddtrace.llmobs._constants import EXPERIMENTS_OUTPUT
 from ddtrace.llmobs._constants import INPUT_DOCUMENTS
@@ -480,6 +482,20 @@ def _llmobs_tags(span: Span, ml_app: str, session_id: Optional[str] = None) -> L
         existing_tags = span._get_ctx_item(TAGS)
         if existing_tags is not None:
             tags.update(existing_tags)
+
+        # set experiment tags on children spans if the tags do not already exist
+        experiment_id = span.context.get_baggage_item(EXPERIMENT_ID_KEY)
+        if experiment_id and "experiment_id" not in tags:
+            tags["experiment_id"] = experiment_id
+
+        run_id = span.context.get_baggage_item(EXPERIMENT_RUN_ID_KEY)
+        if run_id and "run_id" not in tags:
+            tags["run_id"] = run_id
+
+        run_iteration = span.context.get_baggage_item(EXPERIMENT_RUN_ITERATION_KEY)
+        if run_iteration and "run_iteration" not in tags:
+            tags["run_iteration"] = run_iteration
+
         return ["{}:{}".format(k, v) for k, v in tags.items()]
 
     def _do_annotations(self, span: Span) -> None:
@@ -814,6 +830,7 @@ def experiment(
                 ]
             ]
         ] = None,
+        runs: Optional[int] = 1,
     ) -> Experiment:
         """Initializes an Experiment to run a task on a Dataset and evaluators.
 
@@ -830,6 +847,8 @@ def experiment(
                                    to produce a single value.
                                    Must accept parameters ``inputs``, ``outputs``, ``expected_outputs``,
                                    ``evaluators_results``.
+        :param runs: The number of times to run the experiment, or, run the task for every dataset record the defined
+                     number of times.
         """
         if not callable(task):
             raise TypeError("task must be a callable function.")
@@ -870,6 +889,7 @@ def experiment(
             config=config,
             _llmobs_instance=cls._instance,
             summary_evaluators=summary_evaluators,
+            runs=runs,
         )
 
     @classmethod
@@ -1336,6 +1356,8 @@ def _experiment(
         session_id: Optional[str] = None,
         ml_app: Optional[str] = None,
         experiment_id: Optional[str] = None,
+        run_id: Optional[str] = None,
+        run_iteration: Optional[int] = None,
     ) -> Span:
         """
         Trace an LLM experiment, only used internally by the experiments SDK.
@@ -1354,6 +1376,12 @@ def _experiment(
         if experiment_id:
             span.context.set_baggage_item(EXPERIMENT_ID_KEY, experiment_id)
 
+        if run_id:
+            span.context.set_baggage_item(EXPERIMENT_RUN_ID_KEY, run_id)
+
+        if run_iteration is not None:
+            span.context.set_baggage_item(EXPERIMENT_RUN_ITERATION_KEY, run_iteration)
+
         return span
 
     @classmethod

ddtrace/llmobs/_writer.py

@@ -639,6 +639,7 @@ def experiment_create(
         exp_config: Optional[Dict[str, JSONType]] = None,
         tags: Optional[List[str]] = None,
         description: Optional[str] = None,
+        runs: Optional[int] = 1,
     ) -> Tuple[str, str]:
         path = "/api/unstable/llm-obs/v1/experiments"
         resp = self.request(
@@ -656,6 +657,7 @@ def experiment_create(
                         "config": exp_config or {},
                         "metadata": {"tags": cast(JSONType, tags or [])},
                         "ensure_unique": True,
+                        "run_count": runs,
                     },
                 }
             },

releasenotes/notes/llmobs-dne-experiments-multi-run-ef099e98a5827e49.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    LLM Observability: Experiments can now be run multiple times by using the optional ``runs`` argument, 
+    to assess the true performance of an experiment in the face of the non determinism of LLMs. Use the new ``ExperimentResult`` class' ``runs`` attribute to access the results and summary evaluations by run iteration.
+  - |
+    LLM Observability: Non-root experiment spans are now tagged with experiment ID, run ID, and run iteration tags.
+deprecations:
+  - |
+    LLM Observability: The ``ExperimentResult`` class' ``rows`` and ``summary_evaluations`` attributes are deprecated and will be removed in the next major release. ``ExperimentResult.rows/summary_evaluations`` attributes will only store the results of the first run iteration for multi-run experiments. Use the ``ExperimentResult.runs`` attribute instead to access experiment results and summary evaluations.

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_experiments_3f6922dd-477b-40dd-9fd2-baeaab0542a4_events_post_1b36f8c9.yaml

@@ -0,0 +1,51 @@
+interactions:
+- request:
+    body: '{"data": {"type": "experiments", "attributes": {"scope": "experiments",
+      "metrics": [{"metric_source": "custom", "span_id": "123", "trace_id": "456",
+      "timestamp_ms": 1234, "metric_type": "score", "label": "dummy_evaluator", "score_value":
+      0, "error": null, "tags": ["ddtrace.version:1.2.3", "experiment_id:3f6922dd-477b-40dd-9fd2-baeaab0542a4",
+      "run_id:12345678-abcd-abcd-abcd-123456789012", "run_iteration:1"], "experiment_id":
+      "3f6922dd-477b-40dd-9fd2-baeaab0542a4"}], "tags": ["ddtrace.version:1.2.3",
+      "experiment_id:3f6922dd-477b-40dd-9fd2-baeaab0542a4", "run_id:12345678-abcd-abcd-abcd-123456789012",
+      "run_iteration:1"]}}}'
+    headers:
+      Accept:
+      - '*/*'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Accept-Encoding
+      : - identity
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '626'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Type
+      : - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/experiments/3f6922dd-477b-40dd-9fd2-baeaab0542a4/events
+  response:
+    body:
+      string: ''
+    headers:
+      content-length:
+      - '0'
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      content-type:
+      - application/vnd.api+json
+      date:
+      - Wed, 12 Nov 2025 21:30:20 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 202
+      message: Accepted
+version: 1