From 2136d0ed4455e58e2d3f6644d254d04bcabbe786 Mon Sep 17 00:00:00 2001 From: Rosie Yohannan Date: Fri, 19 Sep 2025 09:38:23 +0100 Subject: [PATCH 1/4] remove extension for now --- antora-playbook.yml | 5 +-- docs/guides/modules/ROOT/nav.adoc | 1 + .../pages/agentic-flaky-test-resolution.adoc | 6 --- extensions/unlisted-pages-extension.js | 40 ------------------- 4 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 extensions/unlisted-pages-extension.js diff --git a/antora-playbook.yml b/antora-playbook.yml index 637fb090f2..f93d96b890 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -64,7 +64,4 @@ asciidoc: urls: latest_version_segment: latest latest_version_segment_strategy: redirect:to - html_extension_style: indexify -antora: - extensions: - - ./extensions/unlisted-pages-extension.js \ No newline at end of file + html_extension_style: indexify \ No newline at end of file diff --git a/docs/guides/modules/ROOT/nav.adoc b/docs/guides/modules/ROOT/nav.adoc index 7aad5d5fef..95f713c48f 100644 --- a/docs/guides/modules/ROOT/nav.adoc +++ b/docs/guides/modules/ROOT/nav.adoc @@ -144,6 +144,7 @@ *** xref:test:test.adoc[Automated testing] *** xref:test:collect-test-data.adoc[Collecting test data] *** xref:insights:insights-tests.adoc[Test Insights] +*** xref:test:agentic-flaky-test-resolution.adoc[Agentic flaky test resolution] ** Testing strategies *** xref:test:testing-llm-enabled-applications-through-evaluations.adoc[Testing LLM-enabled applications through evaluations] *** xref:test:browser-testing.adoc[Browser testing] diff --git a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc b/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc index 1ace2e3201..7de6db35b9 100644 --- a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc +++ b/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc @@ -23,8 +23,6 @@ To get started with CircleCI agents for flaky test fixes, you need to complete s . Install the CircleCI GitHub App in your GitHub organization. . Configure the agent with your preferred settings for run frequency and fix parameters. -For a detailed walkthrough of each step, refer to the xref:guides:agents:setup-flaky-test-agent.adoc[Setting up CircleCI agents for flaky tests] guide. - == How CircleCI agents work CircleCI agents operate through an automated analysis and remediation process that runs independently of your regular CI/CD workflows. @@ -124,7 +122,3 @@ Agent logs are stored in CircleCI and remain available for review through the we If you cannot get your OpenAI organization verified, consider using Anthropic as your model provider instead. Alternatively, contact OpenAI support for assistance with the verification process, or reach out to CircleCI support for alternative options. == Next steps - -* xref:guides:agents:setup-flaky-test-agent.adoc[Set up CircleCI agents for flaky tests] -* xref:guides:testing:collect-test-data.adoc[Learn about collecting test data] -* xref:guides:insights:test-insights.adoc[Understanding test insights and flaky test detection] \ No newline at end of file diff --git a/extensions/unlisted-pages-extension.js b/extensions/unlisted-pages-extension.js deleted file mode 100644 index 9cb1957f4f..0000000000 --- a/extensions/unlisted-pages-extension.js +++ /dev/null @@ -1,40 +0,0 @@ -module.exports.register = function ({ config }) { - const { addToNavigation, unlistedPagesHeading = 'Unlisted Pages' } = config - const logger = this.getLogger('unlisted-pages-extension') - this - .on('navigationBuilt', ({ contentCatalog }) => { - contentCatalog.getComponents().forEach(({ versions }) => { - versions.forEach(({ name: component, version, navigation: nav, url: defaultUrl }) => { - const navEntriesByUrl = getNavEntriesByUrl(nav) - const unlistedPages = contentCatalog - .findBy({ component, version, family: 'page' }) - .filter((page) => page.out) - .reduce((collector, page) => { - if ((page.pub.url in navEntriesByUrl) || page.pub.url === defaultUrl) return collector - logger.warn({ file: page.src, source: page.src.origin }, 'detected unlisted page') - return collector.concat(page) - }, []) - if (unlistedPages.length && addToNavigation) { - nav.push({ - content: unlistedPagesHeading, - items: unlistedPages.map((page) => { - const title = 'navtitle' in page.asciidoc - ? page.asciidoc.navtitle - : (page.src.module === 'ROOT' ? '' : page.src.module + ':') + page.src.relative - return { content: title, url: page.pub.url, urlType: 'internal' } - }), - root: true, - }) - } - }) - }) - }) -} - -function getNavEntriesByUrl (items = [], accum = {}) { - items.forEach((item) => { - if (item.urlType === 'internal') accum[item.url.split('#')[0]] = item - getNavEntriesByUrl(item.items, accum) - }) - return accum -} \ No newline at end of file From 77f862b7275352e9f4df26521a0ea84e6bc98268 Mon Sep 17 00:00:00 2001 From: Rosie Yohannan Date: Tue, 23 Sep 2025 11:47:55 +0100 Subject: [PATCH 2/4] work on prerequs and setip --- .../pages/agentic-flaky-test-resolution.adoc | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc b/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc index 7de6db35b9..d384ec04d5 100644 --- a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc +++ b/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc @@ -5,24 +5,51 @@ CircleCI agents provide automated capabilities to identify and resolve common issues in your CI/CD pipelines. Agents can automatically detect flaky tests and generate fixes to help you ship code with confidence by reducing time spent debugging intermittent failures. +TIP: CircleCI agents are AI agents that you can choose to install into your organization to help our with CI/CD related tasks. + == Introduction -Flaky tests are one of the most frustrating challenges in continuous integration. These tests pass and fail inconsistently, creating uncertainty about code quality and slowing down development workflows. CircleCI agents address this problem by using artificial intelligence to analyze test patterns, identify root causes of flakiness, and propose validated solutions. +Flaky tests are tests that pass and fail inconsistently, creating uncertainty about code quality and slowing down development workflows. CircleCI agents can help address the problem of flaky tests by using artificial intelligence to analyze test patterns, identify root causes of flakiness, and propose validated solutions. + +The agent integrates with your existing CircleCI workflows and GitHub repositories. When configured, it runs automatically on a schedule you define, to monitor your test suite for signs of flakiness. When issues are detected, the agent will go through the following steps: -The agent integrates seamlessly with your existing CircleCI workflows and GitHub repositories. When configured, it runs automatically on a schedule you define, continuously monitoring your test suite for signs of flakiness. When issues are detected, the agent generates potential fixes, validates them through multiple test runs, and creates pull requests with recommended changes. +* Generate potential fixes. +* Validate fixes through multiple test runs in an isolated environment. +* Create pull requests with recommended changes after successful validation. This automation helps development teams focus on building features rather than debugging test infrastructure, improving both productivity and confidence in your deployment pipeline. -== Quickstart +== Set up the CircleCI agent and assign a task + +To get started with CircleCI agents for flaky test fixes, you need to fulfill the following prerequisites and complete several setup steps to get the agent installed in your organization. You can then set up tasks for the agent to run. + +=== Prerequisites + +* Get an API key from either link:https://console.anthropic.com/settings/keys[Anthropic] or link:https://auth.openai.com/log-in[OpenAI] for the agent to process and generate fixes. Your source code is not stored nor used for training purposes by CircleCI. If you are using OpenAI you should also check the following +** Make sure your organization link:https://help.openai.com/en/articles/10910291-api-organization-verification#h_c6efff0719[has gpt-5 model access]. +** Verify your organization. For guidance see the link:https://help.openai.com/en/articles/10910291-api-organization-verification[OpenAI help]. If you cannot get your OpenAI organization verified, see the troubleshooting section #INSERT LINK#. +* Ensure your CircleCI jobs store test results using the `store_test_results` step. Read more about this step in the xref:reference:ROOT:configuration-reference.adoc#storetestresults[configuration reference]. +* Ensure you have the CircleCI GitHub app installed in your GitHub organization. Check menu:Organization Settings[VCS Connections], where you can see if you have the App already installed, or select btn:[Install GitHub App]. The CircleCI agent needs the GitHub App to be installed to be able to recommend fixes and open pull requests. + +=== Setup -To get started with CircleCI agents for flaky test fixes, you need to complete several setup steps: +. In the link:https://app.circleci.com/home[CircleCI web app], choose your organization and select *Organization Settings* from the sidebar. +. Select *Agents* from the sidebar. +. Select btn:[Get started] and then btn:[Continue] when prompted. +. If you followed the prerequisites, you should see a image:guides:ROOT:icons/passed.svg[passed icon, role="no-border"] to indicate you already have the GitHub App installed for your organization, +. Select your AI Model provider (Anthropic or OpenAI). +. Enter your API key for your chosen model provider. +. Select btn:[Next] to complete the setup. -. Obtain an API key from either Anthropic or OpenAI for the agent to process and generate fixes. -. Ensure your CircleCI jobs store test results using the `store_test_results` step. -. Navigate to the CircleCI web app and access your organization settings. -. Install the CircleCI GitHub App in your GitHub organization. -. Configure the agent with your preferred settings for run frequency and fix parameters. +#Screenshots# +=== Assign an agent task + +Once you have CircleCI agents set up for your organization you can start assigning tasks. + +To set up a "Fix flaky tests" task follow these steps: + +#insert steps and screenshots# == How CircleCI agents work CircleCI agents operate through an automated analysis and remediation process that runs independently of your regular CI/CD workflows. @@ -49,7 +76,7 @@ The agent runs the modified test multiple times to confirm consistent passing be When the agent successfully validates a solution, it automatically creates a pull request in your GitHub repository. Each pull request includes detailed information about the changes made and the reasoning behind them. -Pull requests contain code diffs showing exactly what changes the agent recommends, along with logs that explain the agent's analysis and decision-making process. This transparency allows your team to understand and review the proposed fixes before merging. +Pull requests contain code diffs showing what changes the agent recommends, along with logs that explain the agent's analysis and decision-making process. This transparency allows your team to understand and review the proposed fixes before merging. == Using tables @@ -62,7 +89,9 @@ The following table shows the configuration options available when setting up a |Run frequency |How often the agent analyzes and fixes flaky tests -|Weekly +a|* Daily +* Weekly (default) +* Monthly |Maximum tests to fix per run |Limits the number of tests the agent will attempt to fix in a single execution @@ -83,13 +112,7 @@ The following table shows the configuration options available when setting up a == Limitations -CircleCI agents have several current limitations that users should be aware of: - -**Configuration editing**: You cannot directly edit setup scripts or post-run commands once an agent task is created. To modify these settings, you must delete the existing agent task and create a new one. - -**OpenAI logs**: Agent logs are not currently available when using OpenAI as your model provider. This feature is available only when using Anthropic. - -**Platform support**: The agent currently runs only in Linux Machine VM environments with basic software installed by default. +*Agent configuration edits are not available*.You cannot directly edit setup scripts or post-run commands once an agent task is created. To modify these settings, you must delete the existing agent task and create a new one. == Troubleshooting From 560d4fb1c57bdecbb104f09812a0d7fabf0e3194 Mon Sep 17 00:00:00 2001 From: Rosie Yohannan Date: Thu, 25 Sep 2025 11:49:47 +0100 Subject: [PATCH 3/4] update page title and ensure we have all info from Discuss post --- docs/guides/modules/ROOT/nav.adoc | 2 +- .../pages/agentic-flaky-test-resolution.adoc | 147 ----------- .../modules/test/pages/fix-flaky-tests.adoc | 238 ++++++++++++++++++ 3 files changed, 239 insertions(+), 148 deletions(-) delete mode 100644 docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc create mode 100644 docs/guides/modules/test/pages/fix-flaky-tests.adoc diff --git a/docs/guides/modules/ROOT/nav.adoc b/docs/guides/modules/ROOT/nav.adoc index 95f713c48f..b9879d1d82 100644 --- a/docs/guides/modules/ROOT/nav.adoc +++ b/docs/guides/modules/ROOT/nav.adoc @@ -144,7 +144,7 @@ *** xref:test:test.adoc[Automated testing] *** xref:test:collect-test-data.adoc[Collecting test data] *** xref:insights:insights-tests.adoc[Test Insights] -*** xref:test:agentic-flaky-test-resolution.adoc[Agentic flaky test resolution] +*** xref:test:fix-flaky-tests.adoc[Fix flaky tests] ** Testing strategies *** xref:test:testing-llm-enabled-applications-through-evaluations.adoc[Testing LLM-enabled applications through evaluations] *** xref:test:browser-testing.adoc[Browser testing] diff --git a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc b/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc deleted file mode 100644 index d384ec04d5..0000000000 --- a/docs/guides/modules/test/pages/agentic-flaky-test-resolution.adoc +++ /dev/null @@ -1,147 +0,0 @@ -= Agentic flaky test resolution -:page-platform: Cloud -:page-description: Learn about CircleCI agents and how they can automatically identify and fix flaky tests in your CI/CD pipelines. -:experimental: - -CircleCI agents provide automated capabilities to identify and resolve common issues in your CI/CD pipelines. Agents can automatically detect flaky tests and generate fixes to help you ship code with confidence by reducing time spent debugging intermittent failures. - -TIP: CircleCI agents are AI agents that you can choose to install into your organization to help our with CI/CD related tasks. - -== Introduction - -Flaky tests are tests that pass and fail inconsistently, creating uncertainty about code quality and slowing down development workflows. CircleCI agents can help address the problem of flaky tests by using artificial intelligence to analyze test patterns, identify root causes of flakiness, and propose validated solutions. - -The agent integrates with your existing CircleCI workflows and GitHub repositories. When configured, it runs automatically on a schedule you define, to monitor your test suite for signs of flakiness. When issues are detected, the agent will go through the following steps: - -* Generate potential fixes. -* Validate fixes through multiple test runs in an isolated environment. -* Create pull requests with recommended changes after successful validation. - -This automation helps development teams focus on building features rather than debugging test infrastructure, improving both productivity and confidence in your deployment pipeline. - -== Set up the CircleCI agent and assign a task - -To get started with CircleCI agents for flaky test fixes, you need to fulfill the following prerequisites and complete several setup steps to get the agent installed in your organization. You can then set up tasks for the agent to run. - -=== Prerequisites - -* Get an API key from either link:https://console.anthropic.com/settings/keys[Anthropic] or link:https://auth.openai.com/log-in[OpenAI] for the agent to process and generate fixes. Your source code is not stored nor used for training purposes by CircleCI. If you are using OpenAI you should also check the following -** Make sure your organization link:https://help.openai.com/en/articles/10910291-api-organization-verification#h_c6efff0719[has gpt-5 model access]. -** Verify your organization. For guidance see the link:https://help.openai.com/en/articles/10910291-api-organization-verification[OpenAI help]. If you cannot get your OpenAI organization verified, see the troubleshooting section #INSERT LINK#. -* Ensure your CircleCI jobs store test results using the `store_test_results` step. Read more about this step in the xref:reference:ROOT:configuration-reference.adoc#storetestresults[configuration reference]. -* Ensure you have the CircleCI GitHub app installed in your GitHub organization. Check menu:Organization Settings[VCS Connections], where you can see if you have the App already installed, or select btn:[Install GitHub App]. The CircleCI agent needs the GitHub App to be installed to be able to recommend fixes and open pull requests. - -=== Setup - -. In the link:https://app.circleci.com/home[CircleCI web app], choose your organization and select *Organization Settings* from the sidebar. -. Select *Agents* from the sidebar. -. Select btn:[Get started] and then btn:[Continue] when prompted. -. If you followed the prerequisites, you should see a image:guides:ROOT:icons/passed.svg[passed icon, role="no-border"] to indicate you already have the GitHub App installed for your organization, -. Select your AI Model provider (Anthropic or OpenAI). -. Enter your API key for your chosen model provider. -. Select btn:[Next] to complete the setup. - -#Screenshots# - -=== Assign an agent task - -Once you have CircleCI agents set up for your organization you can start assigning tasks. - -To set up a "Fix flaky tests" task follow these steps: - -#insert steps and screenshots# -== How CircleCI agents work - -CircleCI agents operate through an automated analysis and remediation process that runs independently of your regular CI/CD workflows. - -=== Test analysis and detection - -The agent continuously monitors test results stored in CircleCI to identify patterns of flakiness. It analyzes historical test data to distinguish between genuine failures caused by code issues and intermittent failures that indicate flaky behavior. Tests are flagged as flaky when they show inconsistent pass/fail patterns across multiple runs with the same code. - -The detection process considers factors such as failure frequency, timing patterns, and error message consistency. This helps the agent focus on tests that genuinely exhibit flaky behavior rather than tests that fail consistently due to code problems. - -=== Solution generation - -When a flaky test is identified, the agent generates potential solutions based on common flaky test patterns and best practices. The agent can create multiple solution approaches for each test, allowing it to try different fixes if the first attempt does not resolve the issue. - -Solutions may include adding explicit waits, improving element selectors, handling race conditions, or stabilizing test data setup. The agent tailors its recommendations to the specific failure patterns observed in your test. - -=== Validation process - -Before proposing any changes, the agent validates potential solutions through multiple test runs in an isolated environment. This validation process ensures that proposed fixes actually resolve the flakiness without breaking existing functionality. - -The agent runs the modified test multiple times to confirm consistent passing behavior. Only solutions that demonstrate reliable improvement are included in pull requests. If validation fails or the agent lacks confidence in a solution, no pull request is created, but analysis logs remain available for review. - -=== Pull request creation - -When the agent successfully validates a solution, it automatically creates a pull request in your GitHub repository. Each pull request includes detailed information about the changes made and the reasoning behind them. - -Pull requests contain code diffs showing what changes the agent recommends, along with logs that explain the agent's analysis and decision-making process. This transparency allows your team to understand and review the proposed fixes before merging. - -== Using tables - -The following table shows the configuration options available when setting up a CircleCI agent: - -.Agent configuration options -[cols="1,2,1"] -|=== -|Setting |Description |Default - -|Run frequency -|How often the agent analyzes and fixes flaky tests -a|* Daily -* Weekly (default) -* Monthly - -|Maximum tests to fix per run -|Limits the number of tests the agent will attempt to fix in a single execution -|5 - -|Number of solutions to try per test -|How many different fix approaches the agent will generate for each flaky test -|3 - -|Number of validation runs per test -|How many times the agent runs a test to validate that a fix works consistently -|10 - -|Maximum concurrent open PRs -|Limits the number of pull requests the agent can have open at one time -|5 -|=== - -== Limitations - -*Agent configuration edits are not available*.You cannot directly edit setup scripts or post-run commands once an agent task is created. To modify these settings, you must delete the existing agent task and create a new one. - -== Troubleshooting - -=== Unable to run verification tests - -If the agent cannot run verification tests, this typically indicates missing dependencies or environment configuration issues. The agent runs in a Linux Machine VM and may need additional software to execute your tests properly. - -Check the agent task logs in the CircleCI web app by expanding all log sections and searching for "attempt" to see what the agent tried to do. Use the setup script feature during agent configuration to install required dependencies or environment requirements. - -Consider adding a markdown file to your repository with instructions for running tests locally. The agent will automatically detect and use this information to better understand your test environment. - -=== Agent execution errors - -When the agent encounters execution errors, review the logs to understand what went wrong. Common issues include missing API keys, insufficient permissions, or problems accessing your repository. - -Verify that your API key is valid and has the necessary permissions for your chosen model provider. Ensure that the CircleCI GitHub App has been installed with appropriate access to your organization and repositories. - -== Frequently asked questions - -=== Does CircleCI use my data to train AI models? - -No, CircleCI does not store your source code or use it for training purposes. The agent processes your code temporarily to generate fixes but does not retain or share this information with model providers for training. - -=== How long are agent logs stored? - -Agent logs are stored in CircleCI and remain available for review through the web app. The specific retention period for these logs follows CircleCI's standard data retention policies. - -=== What if my OpenAI organization cannot be verified? - -If you cannot get your OpenAI organization verified, consider using Anthropic as your model provider instead. Alternatively, contact OpenAI support for assistance with the verification process, or reach out to CircleCI support for alternative options. - -== Next steps diff --git a/docs/guides/modules/test/pages/fix-flaky-tests.adoc b/docs/guides/modules/test/pages/fix-flaky-tests.adoc new file mode 100644 index 0000000000..e9bdb17da7 --- /dev/null +++ b/docs/guides/modules/test/pages/fix-flaky-tests.adoc @@ -0,0 +1,238 @@ += Fix flaky tests +:page-platform: Cloud +:page-description: Learn about CircleCI agents and how they can automatically identify and fix flaky tests in your CI/CD pipelines. +:experimental: + +Use CircleCI Chunk to automatically identify and present resolutions to flaky tests in your CI/CD pipelines. + +CircleCI Chunk provides automated capabilities to identify and resolve common issues in your CI/CD pipelines. Chunk can automatically detect flaky tests and generate fixes to help you reduce the time spent debugging intermittent failures. + +TIP: CircleCI Chunk is an AI agent that you can choose to install into your organization to help with CI/CD related tasks. + +== Introduction + +Flaky tests are tests that pass and fail inconsistently. Flaky tests create uncertainty about code quality and slow down development workflows. Chunk can help address the problem of flaky tests by using artificial intelligence to analyze test patterns, identify root causes of flakiness, and propose validated solutions. + +Chunk integrates with your existing CircleCI workflows and GitHub repositories. When configured, Chunk tasks run automatically on a schedule you define, to monitor your test suite for signs of flakiness. When issues are detected, Chunk will go through the following steps: + +* Generate potential fixes. +* Validate fixes through multiple test runs in an isolated environment. +* Create pull requests with recommended changes after successful validation. + +== Set up the CircleCI Chunk and assign a task + +To get started with automating flaky test fixes, you need to fulfill the following prerequisites and complete several setup steps to get the Chunk installed in your organization. You can then assign tasks for Chunk to run. + +=== Prerequisites + +* You will need an API key from either link:https://console.anthropic.com/settings/keys[Anthropic] or link:https://auth.openai.com/log-in[OpenAI] for Chunk to process and generate fixes. Your source code is not stored nor used for training purposes by CircleCI. If you are using OpenAI you should also check the following: +** Make sure your organization link:https://help.openai.com/en/articles/10910291-api-organization-verification#h_c6efff0719[has gpt-5 model access]. +** Verify your organization. For guidance see the link:https://help.openai.com/en/articles/10910291-api-organization-verification[OpenAI help]. If you cannot get your OpenAI organization verified, see the troubleshooting item <>. +* Ensure your CircleCI jobs store test results using the `store_test_results` step. Read more about this step in the xref:reference:ROOT:configuration-reference.adoc#storetestresults[configuration reference]. +* Ensure you have the CircleCI GitHub app installed in your GitHub organization. Check menu:Organization Settings[VCS Connections], where you can see if you have the App already installed, or select btn:[Install GitHub App]. The CircleCI agent needs the GitHub App to be installed to be able to recommend fixes and open pull requests. +* Make sure you are following projects in your organization that do have flaky tests to fix. CircleCI identifies flaky tests in your CI/CD pipelines on the *Tests* tab for workflows in the Insights dashboard (menu:Insights[Select project > Select workflow > Tests]). + +=== Setup + +. In the link:https://app.circleci.com/home[CircleCI web app], choose your organization and select *Organization Settings* from the sidebar. +. Select *Chunk Tasks* from the sidebar. +. Select btn:[Get started] and then btn:[Continue] when prompted. +. If you followed the prerequisites, you should see a image:guides:ROOT:icons/passed.svg[passed icon, role="no-border"] to indicate you already have the GitHub App installed for your organization. If not use the btn:[Install CircleCI GitHub App] button to install it. +. Select your AI Model provider (Anthropic or OpenAI). +. Enter your API key for your chosen model provider. +. Select btn:[Next] to complete the setup. + +#Screenshots# + +=== Assign a task + +Once you have Chunk installed and set up for your organization you can start assigning tasks. + +To set up a "Fix flaky tests" task follow these steps: + +. Select the project you want to assign the task to. +. Choose a run frequency (daily, weekly, monthly). +. Choose the number of tests you want Chunk to try to fic per run, between one and three. +. Choose a maximum for the number of solutions you want Chunk to try for each test, between one and three. +. Choose a number of validation runs to allow per test between one and 20. +. Choose a maximum number of concurrent open PRs for flakey test fixes. You can choose between one and 20, or the default, "Unlimited". +. In the "Custom settings" section you can add a setup script for the Chunk agent to run before each run. Youc an also add post-run commands to run after each run. +. Select btn:[Start task] to complete the setup. + +When you select btn:[Start task] Chunk will start running immediately and follow the schedule you set up.. + +#insert screenshots# + +== How CircleCI agents work + +CircleCI agents operate through an automated analysis and remediation process that runs independently of your regular CI/CD workflows. + +=== Test analysis and detection + +The agent continuously monitors test results stored in CircleCI to identify patterns of flakiness. It analyzes historical test data to distinguish between genuine failures caused by code issues and intermittent failures that indicate flaky behavior. Tests are flagged as flaky when they show inconsistent pass/fail patterns across multiple runs with the same code. + +The detection process considers factors such as failure frequency, timing patterns, and error message consistency. This helps the agent focus on tests that genuinely exhibit flaky behavior rather than tests that fail consistently due to code problems. + +=== Solution generation + +When a flaky test is identified, the agent generates potential solutions based on common flaky test patterns and best practices. The agent can create multiple solution approaches for each test, allowing it to try different fixes if the first attempt does not resolve the issue. + +Solutions may include adding explicit waits, improving element selectors, handling race conditions, or stabilizing test data setup. The agent tailors its recommendations to the specific failure patterns observed in your test. + +=== Validation process + +Before proposing any changes, the agent validates potential solutions through multiple test runs in an isolated environment. This validation process ensures that proposed fixes actually resolve the flakiness without breaking existing functionality. + +The agent runs the modified test multiple times to confirm consistent passing behavior. Only solutions that demonstrate reliable improvement are included in pull requests. If validation fails or the agent lacks confidence in a solution, no pull request is created, but analysis logs remain available for review. + +=== Pull request creation + +When the agent successfully validates a solution, it automatically creates a pull request in your GitHub repository. Each pull request includes detailed information about the changes made and the reasoning behind them. + +Pull requests contain code diffs showing what changes the agent recommends, along with logs that explain the agent's analysis and decision-making process. This transparency allows your team to understand and review the proposed fixes before merging. + +== The Chunk tasks dashboard + +Once Chunk tasks are set up you can view an activity timeline in the Chunk tasks dashboard. + +#screenshot# + +Once a fix is verified you can select the "PR opened" row to view the task overview, which includes the following information: + +* Summary of the fix +* Root cause of the flakiness +* Details of the proposed fix +* Details of the level of verification achieved + +#Screenshot# + +You also get a code diff of the proposed fix along with logs of the decision process presented as a conversation between "User" (Chunk) and "Assistant" (AI modelprovider) to help you understand Chunk's reasoning and analysis process. + +If Chunk did not manage to reproduce the flakiness, lacks confidence in fixes, or the test runs fail, no PR is created but you can still view the logs to review the analysis and decision process. + +== Flaky test fix configuration options + +The following table shows the configuration options available when setting up a CircleCI agent: + +.Agent configuration options +[cols="1,2,1"] +|=== +|Setting |Description |Default + +|Run frequency +|How often the agent analyzes and fixes flaky tests +a|* Daily +* Weekly (default) +* Monthly + +|Maximum tests to fix per run +|Limits the number of tests the agent will attempt to fix in a single execution +| 1, 2, 3 (default) + +|Number of solutions to try per test +|How many different fix approaches the agent will generate for each flaky test +|1 (default), 2, 3 + +|Number of validation runs per test +|How many times the agent runs a test to validate that a fix works consistently +|1-20. 10 is the default. + +|Maximum concurrent open PRs +|Limits the number of pull requests the agent can have open at one time +|1-20 or "Unlimited" (default). +|=== + +== Limitations + +*Agent configuration edits are not available*.You cannot directly edit setup scripts or post-run commands once an agent task is created. To modify these settings, you must delete the existing agent task and create a new one. + +== Troubleshooting + +=== Unable to run verification tests + +Chunk runs in a xref:execution-managed:using-linuxvm.adoc[Linux VM] with link:https://discuss.circleci.com/t/ubuntu-20-04-22-04-24-04-q4-edge-release/52429[basic software installed by default]. To verify that a proposed fix resolves flakiness, it re-runs the affected test several times. To do this, the agent may install additional software needed to set up the test environment, using clues from your CircleCI configuration file to determine how to run the tests. + +You can view these attempts in the CircleCI web app by opening the Chunk task from the timeline, then select *Logs* and select the btn:[Expand All] option, then search for “attempt.” This will take you to the section where the agent is trying to run the tests. + +To improve verification success, create an "agent environment" CircleCI YAML file. Copy the environment setup parts of your existing CircleCI configuration into a dedicated file for Chunk. + +* Name the file `cci-agent-setup.yml` and save it to your `.circleci` directory on your default branch. +* `cci-agent-setup.yml` needs to include a single workflow (the name of the workflow can be anything you want) with a single job named `cci-agent-setup`. The `cci-agent-setup` job needs to set up your environment for Chunk to use. You do not need to include any steps to run tests, this is purely for environment setup. ++ +.Example config file for cci-agent-setup.yml +[source,yaml] +---- +version: 2.1 +workflows: + main: + jobs: + - cci-agent-setup +jobs: + cci-agent-setup: + docker: + - image: cimg/python:3.12 + - image: cimg/postgres:15.3 + steps: + - checkout + - run: + name: Hello World + command: | + echo "Hello, World!" + # insert more environment setup here +---- + + +Also consider including a markdown file, named `claude.md` or `agents.md` at the root of your repository with instructions for running tests. The agent should pick this up automatically. + + +=== Invalid OpenAI modal specified + +If you get the following error: + +[source,shell] +Invalid OpenAI model specified. Please check the model name and ensure it is available for your account. + +You will need to make sure your organization has GPT-5 access. To verify this in link:https://platform.openai.com/settings/organization/general[OpenAI Platform], follow these steps: + +. Switch to the project you want to check in the top left dropdown. +. Go to menu:Settings[Limits] in the left-hand menu. This page shows the models and rate limits for your project. `gpt-5` will be listed if you have access. + +=== I cannot get my OpenAI organization verified + +If organization verification is not possible, you can bypass this requirement by adding an environment variable to your `circleci-agents` context, as follows: + +. In the CircleCI web app, go to menu:Organization Settings[Contexts]. +. Use the search to find the `circleci-agents` context. Select it by name to open configuration options. +. Scroll down to the "Environment variables" section. +. Select btn:[Add environment variable] to enter the variable name and value. +** Under "Environment variable name", enter `CCI_AGENT_OPENAI_MODEL`. +** Under "Value", enter `gtp-5-nano`. + +=== Verification required error + +If you get the following error inside an agent task, this indicates that your Open AI organization verification is pending. + +[source,shell] +OpenAI organization verification required. Please verify your organization at https://platform.openai.com/settings/organization/generaland see our community forum for more debugging help. + +To fix this issues, head to link:https://platform.openai.com/settings/organization/general[OpenAI Platform], navigate to menu:General[Organization Settings] and select btn:[Verify Organization]. Then follow the steps to get your organization verified. + +=== Action required error + +If you get the following error inside an agent task, this indicates that your Open AI organization verification is pending. + +[source,shell] +Action required - agent execution error +The agent ran into an error while executing this task. See our community forum for how to solve this error. + +Contact link:https://support.circleci.com/hc/en-us/[CircleCI support] for assistance. + +== Frequently asked questions + +=== Does CircleCI use my data to train AI models? + +No, CircleCI does not store your source code or use it for training purposes. Chunk processes your code temporarily to generate fixes but does not retain or share this information with model providers for training. + +=== How long are agent logs stored? + +Chunks logs are stored by CircleCI for 90 days. 90 days is a fixed retention period that applies to all organizations, link:https://support.circleci.com/hc/en-us/articles/5645222646939-Cloud-Data-Retention-Policy-and-Settings[regardless of your plan’s standard data retention policy]. After 90 days, logs are automatically deleted to keep your workspace at optimal performance. \ No newline at end of file From c292fae36c557c9388f08fca83154927077cde07 Mon Sep 17 00:00:00 2001 From: Rosie Yohannan Date: Thu, 25 Sep 2025 11:55:25 +0100 Subject: [PATCH 4/4] revert change to playbook --- antora-playbook.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/antora-playbook.yml b/antora-playbook.yml index f93d96b890..637fb090f2 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -64,4 +64,7 @@ asciidoc: urls: latest_version_segment: latest latest_version_segment_strategy: redirect:to - html_extension_style: indexify \ No newline at end of file + html_extension_style: indexify +antora: + extensions: + - ./extensions/unlisted-pages-extension.js \ No newline at end of file