chore: (studio) set up infrastructure for cypress in cypress tests for cloud studio (#31621)

* internal: (studio) set up infrastructure for cypress in cypress tests for cloud studio

* remove local code

* fix build

* fix build

* extract cloud env to a constant

* Revert "extract cloud env to a constant"

This reverts commit 8e9c165.

Author: ryanthemanuel

guides/README.md

@@ -19,7 +19,9 @@ For general contributor information, check out [`CONTRIBUTING.md`](../CONTRIBUTI
 * [Error handling](./error-handling.md)
 * [GraphQL Subscriptions - Overview and Test Guide](./graphql-subscriptions.md)
 * [Patching packages](./patch-package.md)
+* [Protocol development](./protocol-development.md)
 * [Release process](./release-process.md)
+* [Studio development](./studio-development.md)
 * [Testing other projects](./testing-other-projects.md)
 * [Testing strategy and style guide (draft)](./testing-strategy-and-styleguide.md)
 * [Writing cross-platform JavaScript](./writing-cross-platform-javascript.md)

guides/studio-development.md

@@ -42,3 +42,59 @@ or to reference a local `cypress_services` repo:
 ```sh
 CYPRESS_LOCAL_STUDIO_PATH=<path-to-cypress-services/app/studio/dist/development-directory> yarn gulp downloadStudioTypes
 ```
+
+## Testing
+
+### Unit/Component Testing
+
+The code that supports cloud Studio and lives in the `cypress` monorepo is unit and component tested in a similar fashion to the rest of the code in the repo. See the [contributing guide](https://github.com/cypress-io/cypress/blob/ad353fcc0f7fdc51b8e624a2a1ef4e76ef9400a0/CONTRIBUTING.md?plain=1#L366) for more specifics.
+
+The code that supports cloud Studio and lives in the `cypress-services` monorepo has unit and component tests that live alongside the code in that monorepo.
+
+### Cypress in Cypress Testing
+
+Several helpers are provided to facilitate testing cloud Studio using Cypress in Cypress tests. The [helper file](https://github.com/cypress-io/cypress/blob/ad353fcc0f7fdc51b8e624a2a1ef4e76ef9400a0/packages/app/cypress/e2e/studio/helper.ts) provides a method, `launchStudio` that:
+
+1. Loads a project (by default [`experimental-studio`](https://github.com/cypress-io/cypress/tree/develop/system-tests/projects/experimental-studio)).
+2. Navigates to the appropriate spec (by default `specName.cy.js`).
+3. Enters Studio either by creating a new test or entering from an existing test via the `createNewTest` parameter
+4. Waits for the test to finish executing again in Studio mode.
+
+The above steps actually download the studio code from the cloud and use it for the test. Note that `experimental-studio` is set up to be a `canary` project so it will always get the latest and greatest of the cloud Studio code, whether or not it has been fully promoted to production. Note that this means that if you are writing Cypress in Cypress tests that depend on new functionality delivered from the cloud, the Cypress in Cypress tests cannot be merged until the code lands and is built in the cloud. Local development is still possible however by setting `process.env.CYPRESS_LOCAL_STUDIO_PATH` to your local studio path where we enable studio [here](https://github.com/cypress-io/cypress/blob/develop/packages/frontend-shared/cypress/e2e/e2ePluginSetup.ts#L424).
+
+In order to properly engage with Studio AI, we choose to simulate the cloud interactions that enable it via something like:
+
+```js
+cy.mockNodeCloudRequest({
+  url: '/studio/testgen/n69px6/enabled',
+  method: 'get',
+  body: { enabled: true },
+})
+```
+
+To ensure that we get the same results from our Studio AI calls every time, we simulate them via something like:
+
+```js
+const aiOutput = 'cy.get(\'button\').should(\'have.text\', \'Increment\')'
+cy.mockNodeCloudStreamingRequest({
+  url: '/studio/testgen/n69px6/generate',
+  method: 'post',
+  body: { recommendations: [{ content: aiOutput }] },
+})
+```
+
+The above two helpers actually mock out the Node requests so we still test the interface between the browser and node with these tests.
+
+Also, since protocol does not work properly on the inner Cypress of Cypress in Cypress tests, we choose to create a dummy protocol which means we need to provide a simulated CDP full snapshot that will be sent to AI via something like:
+
+```js
+cy.mockStudioFullSnapshot({
+  id: 1,
+  nodeType: 1,
+  nodeName: 'div',
+  localName: 'div',
+  nodeValue: 'div',
+  children: [],
+  shadowRoots: [],
+})
+```

packages/app/cypress/e2e/studio/studio.cy.ts

@@ -167,6 +167,74 @@ describe('studio functionality', () => {
 
       cy.percySnapshot()
     })
+
+    it('opens a cloud studio session with AI enabled', () => {
+      cy.mockNodeCloudRequest({
+        url: '/studio/testgen/n69px6/enabled',
+        method: 'get',
+        body: { enabled: true },
+      })
+
+      const aiOutput = 'cy.get(\'button\').should(\'have.text\', \'Increment\')'
+
+      cy.mockNodeCloudStreamingRequest({
+        url: '/studio/testgen/n69px6/generate',
+        method: 'post',
+        body: { recommendations: [{ content: aiOutput }] },
+      })
+
+      cy.mockStudioFullSnapshot({
+        id: 1,
+        nodeType: 1,
+        nodeName: 'div',
+        localName: 'div',
+        nodeValue: 'div',
+        children: [],
+        shadowRoots: [],
+      })
+
+      const deferred = pDefer()
+
+      loadProjectAndRunSpec({ enableCloudStudio: true })
+
+      cy.findByTestId('studio-panel').should('not.exist')
+
+      cy.intercept('/cypress/e2e/index.html', () => {
+        // wait for the promise to resolve before responding
+        // this will ensure the studio panel is loaded before the test finishes
+        return deferred.promise
+      }).as('indexHtml')
+
+      cy.contains('visits a basic html page')
+      .closest('.runnable-wrapper')
+      .findByTestId('launch-studio')
+      .click()
+
+      // regular studio is not loaded until after the test finishes
+      cy.get('[data-cy="hook-name-studio commands"]').should('not.exist')
+      // cloud studio is loaded immediately
+      cy.findByTestId('studio-panel').then(() => {
+        // check for the loading panel from the app first
+        cy.get('[data-cy="loading-studio-panel"]').should('be.visible')
+        // we've verified the studio panel is loaded, now resolve the promise so the test can finish
+        deferred.resolve()
+      })
+
+      cy.wait('@indexHtml')
+
+      // Studio re-executes spec before waiting for commands - wait for the spec to finish executing.
+      cy.waitForSpecToFinish()
+
+      // Verify the studio panel is still open
+      cy.findByTestId('studio-panel')
+      cy.get('[data-cy="hook-name-studio commands"]')
+
+      // Verify that AI is enabled
+      cy.get('[data-cy="ai-status-text"]').should('contain.text', 'Enabled')
+
+      // Verify that the AI output is correct
+      cy.get('[data-cy="studio-ai-output-textarea"]').should('contain.text', aiOutput)
+    })
   })
 
   it('updates an existing test with an action', () => {

packages/frontend-shared/cypress/e2e/e2ePluginSetup.ts

@@ -24,20 +24,24 @@ import path from 'path'
 import execa from 'execa'
 import _ from 'lodash'
 
-import type { CyTaskResult, OpenGlobalModeOptions, RemoteGraphQLBatchInterceptor, RemoteGraphQLInterceptor, ResetOptionsResult, WithCtxInjected, WithCtxOptions } from '../support/e2e'
+import type { CyTaskResult, OpenGlobalModeOptions, RemoteGraphQLBatchInterceptor, RemoteGraphQLInterceptor, ResetOptionsResult, WithCtxInjected, WithCtxOptions, MockNodeCloudRequestOptions, MockNodeCloudStreamingRequestOptions } from '../support/e2e'
 import { fixtureDirs } from '@tooling/system-tests'
 import * as inspector from 'inspector'
 // tslint:disable-next-line: no-implicit-dependencies - requires cypress
 import sinonChai from '@cypress/sinon-chai'
 import sinon from 'sinon'
 import fs from 'fs-extra'
+import nock from 'nock'
 import { CYPRESS_REMOTE_MANIFEST_URL, NPM_CYPRESS_REGISTRY_URL } from '@packages/types'
 
 import { CloudQuery } from '@packages/graphql/test/stubCloudTypes'
 import pDefer from 'p-defer'
+import { Readable } from 'stream'
 
 const pkg = require('@packages/root')
 
+const dummyProtocolPath = path.join(__dirname, '../fixtures/dummy-protocol.js')
+
 export interface InternalOpenProjectCapabilities {
   cloudStudio: boolean
 }
@@ -196,6 +200,8 @@ async function makeE2ETasks () {
      */
     __internal__afterEach () {
       delete process.env.CYPRESS_ENABLE_CLOUD_STUDIO
+      delete process.env.CYPRESS_IN_CYPRESS_MOCK_FULL_SNAPSHOT
+      nock.cleanAll()
 
       return null
     },
@@ -422,6 +428,9 @@ async function makeE2ETasks () {
     async __internal_openProject ({ argv, projectName, capabilities }: InternalOpenProjectArgs): Promise<ResetOptionsResult> {
       if (capabilities.cloudStudio) {
         process.env.CYPRESS_ENABLE_CLOUD_STUDIO = 'true'
+        // Cypress in Cypress testing breaks pretty heavily in terms of the inner Cypress's protocol. For now, we essentially
+        // disable the protocol by using a dummy protocol that does nothing and allowing tests to mock studio full snapshots as needed.
+        process.env.CYPRESS_LOCAL_PROTOCOL_PATH = dummyProtocolPath
       }
 
       let projectMatched = false
@@ -463,6 +472,52 @@ async function makeE2ETasks () {
         e2eServerPort: ctx.coreData.servers.appServerPort,
       }
     },
+    __internal_mockStudioFullSnapshot (fullSnapshot: Record<string, any>) {
+      // This is the outlet to provide a mock full snapshot for studio tests.
+      // This is necessary because protocol does not capture things properly in the inner Cypress
+      // when running in Cypress in Cypress.
+      process.env.CYPRESS_IN_CYPRESS_MOCK_FULL_SNAPSHOT = JSON.stringify(fullSnapshot)
+
+      return null
+    },
+    __internal_mockNodeCloudRequest ({ url, method, body }: MockNodeCloudRequestOptions) {
+      const nocked = nock('https://cloud.cypress.io', {
+        allowUnmocked: true,
+      })
+
+      nocked[method](url).reply(200, body)
+
+      return null
+    },
+    __internal_mockNodeCloudStreamingRequest ({ url, method, body }: MockNodeCloudStreamingRequestOptions) {
+      const nocked = nock('https://cloud.cypress.io', {
+        allowUnmocked: true,
+      })
+
+      // This format is exactly what is expected by our cloud streaming requests (currently just our
+      // interactions with studio AI). Note that this does not replicate how the event streaming
+      // works exactly, but it is good enough for these Cypress in Cypress purposes and we test
+      // the full functionality with all edge cases alongside the Studio cloud code.
+      nocked[method](url).reply(200, () => {
+        const stream = new Readable({
+          read () {
+            this.push('event: chunk\n')
+            this.push(`data: ${JSON.stringify(body)}\n\n`)
+            this.push('event: end\n')
+            this.push('data: \n\n')
+            this.push(null)
+          },
+        })
+
+        return stream
+      }, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        Connection: 'keep-alive',
+      })
+
+      return null
+    },
     async __internal_withCtx (obj: WithCtxObj): Promise<CyTaskResult<any>> {
       const options: WithCtxInjected = {
         ...obj.options,

packages/frontend-shared/cypress/fixtures/dummy-protocol.js

@@ -0,0 +1,58 @@
+const { Readable } = require('stream')
+
+class AppCaptureProtocol {
+  uploadStallSamplingInterval () {
+    return 0
+  }
+  cdpReconnect () {
+    return Promise.resolve()
+  }
+  responseEndedWithEmptyBody (options) {
+    return
+  }
+  responseStreamTimedOut (options) {
+    return
+  }
+  getDbMetadata () {
+    return {
+      offset: 0,
+      size: 0,
+    }
+  }
+  responseStreamReceived (options) {
+    return Readable.from([])
+  }
+  beforeSpec ({ workingDirectory, archivePath, dbPath, db }) {
+  }
+  addRunnables (runnables) {
+  }
+  commandLogAdded (log) {
+  }
+  commandLogChanged (log) {
+  }
+  viewportChanged (input) {
+  }
+  urlChanged (input) {
+  }
+  beforeTest (test) {
+    return Promise.resolve()
+  }
+  preAfterTest (test, options) {
+    return Promise.resolve()
+  }
+  afterTest (test) {
+    return Promise.resolve()
+  }
+  afterSpec () {
+    return Promise.resolve({ durations: {} })
+  }
+  connectToBrowser (cdpClient) {
+    return Promise.resolve()
+  }
+  pageLoading (input) {
+  }
+  resetTest (testId) {
+  }
+}
+
+module.exports = { AppCaptureProtocol }

packages/frontend-shared/cypress/support/e2e.ts

@@ -12,6 +12,7 @@ import type sinon from 'sinon'
 import type pDefer from 'p-defer'
 import 'cypress-plugin-tab'
 import type { Response } from 'cross-fetch'
+import type nock from 'nock'
 
 import type { E2ETaskMap, InternalOpenProjectCapabilities } from '../e2e/e2ePluginSetup'
 import { installCustomPercyCommand } from './customPercyCommand'
@@ -80,6 +81,18 @@ export interface FindBrowsersOptions {
   filter?(browser: Browser): boolean
 }
 
+export interface MockNodeCloudRequestOptions {
+  url: string
+  method: string
+  body: nock.Body
+}
+
+export interface MockNodeCloudStreamingRequestOptions {
+  url: string
+  method: string
+  body: nock.Body
+}
+
 export interface ValidateExternalLinkOptions {
   /**
    * The user-visible descriptor for the link. If omitted, the href
@@ -184,6 +197,20 @@ declare global {
        * Get the AUT <iframe>. Useful for Cypress in Cypress tests.
        */
       getAutIframe(): Chainable<JQuery<HTMLIFrameElement>>
+      /**
+       * Mocks a studio full snapshot as if it were captured in the inner Cypress's protocol.
+       * This is necessary because protocol does not capture things properly in the inner Cypress
+       * when running in Cypress in Cypress.
+       */
+      mockStudioFullSnapshot(fullSnapshot: Record<string, any>): void
+      /**
+       * Mocks a node cloud request
+       */
+      mockNodeCloudRequest(options: { url: string, method: string, body: nock.Body }): void
+      /**
+       * Mocks a node cloud streaming request
+       */
+      mockNodeCloudStreamingRequest(options: { url: string, method: string, body: nock.Body }): void
     }
 
   }
@@ -260,6 +287,24 @@ function openProject (projectName: WithPrefix<ProjectFixtureDir>, argv: string[]
   })
 }
 
+function mockStudioFullSnapshot (fullSnapshot: Record<string, any>) {
+  return logInternal({ name: 'mockStudioFullSnapshot' }, () => {
+    return taskInternal('__internal_mockStudioFullSnapshot', fullSnapshot)
+  })
+}
+
+function mockNodeCloudRequest (options: MockNodeCloudRequestOptions) {
+  return logInternal({ name: 'mockNodeCloudRequest' }, () => {
+    return taskInternal('__internal_mockNodeCloudRequest', options)
+  })
+}
+
+function mockNodeCloudStreamingRequest (options: MockNodeCloudStreamingRequestOptions) {
+  return logInternal({ name: 'mockNodeCloudStreamingRequest' }, () => {
+    return taskInternal('__internal_mockNodeCloudStreamingRequest', options)
+  })
+}
+
 function startAppServer (mode: 'component' | 'e2e' = 'e2e', options: { skipMockingPrompts: boolean } = { skipMockingPrompts: false }) {
   const { name, family } = Cypress.browser
 
@@ -602,6 +647,9 @@ Cypress.Commands.add('remoteGraphQLInterceptBatched', remoteGraphQLInterceptBatc
 Cypress.Commands.add('findBrowsers', findBrowsers)
 Cypress.Commands.add('tabUntil', tabUntil)
 Cypress.Commands.add('validateExternalLink', { prevSubject: ['optional', 'element'] }, validateExternalLink)
+Cypress.Commands.add('mockStudioFullSnapshot', mockStudioFullSnapshot)
+Cypress.Commands.add('mockNodeCloudRequest', mockNodeCloudRequest)
+Cypress.Commands.add('mockNodeCloudStreamingRequest', mockNodeCloudStreamingRequest)
 
 installCustomPercyCommand({
   elementOverrides: {

packages/frontend-shared/package.json

@@ -89,6 +89,7 @@
     "lodash": "4.17.21",
     "markdown-it": "13.0.1",
     "modern-normalize": "1.1.0",
+    "nock": "13.2.9",
     "p-defer": "^3.0.0",
     "patch-package": "8.0.0",
     "postcss": "^8.4.22",